{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "---\n",
    "## BigDL-Nano Resnet example on CIFAR10 dataset\n",
    "---\n",
    "This example illustrates how to apply bigdl-nano optimizations on a image recognition case based on pytorch-lightning framework. The basic image recognition module is implemented with Lightning and trained on [CIFAR10](https://www.cs.toronto.edu/~kriz/cifar.html) image recognition Benchmark dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/nanoPytorch/lib/python3.7/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from time import time\n",
    "import os\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision\n",
    "from pl_bolts.datamodules import CIFAR10DataModule\n",
    "from pl_bolts.transforms.dataset_normalizations import cifar10_normalization\n",
    "from pytorch_lightning import LightningModule, seed_everything\n",
    "from pytorch_lightning.callbacks import LearningRateMonitor, ModelCheckpoint\n",
    "from pytorch_lightning.loggers import TensorBoardLogger\n",
    "from torch.optim.lr_scheduler import OneCycleLR\n",
    "from torchmetrics.functional import accuracy\n",
    "from bigdl.nano.pytorch.trainer import Trainer\n",
    "from bigdl.nano.pytorch.vision import transforms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### CIFAR10 Data Module\n",
    "---\n",
    "Import the existing data module from bolts and modify the train and test transforms.\n",
    "You could access [CIFAR10](https://www.cs.toronto.edu/~kriz/cifar.html) for a view of the whole dataset.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def prepare_data(data_path, batch_size, num_workers):\n",
    "    train_transforms = transforms.Compose(\n",
    "        [\n",
    "            transforms.RandomCrop(32, 4),\n",
    "            transforms.RandomHorizontalFlip(),\n",
    "            transforms.ToTensor(),\n",
    "            cifar10_normalization()\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    test_transforms = transforms.Compose(\n",
    "        [\n",
    "            transforms.ToTensor(),\n",
    "            cifar10_normalization()\n",
    "        ]\n",
    "    )\n",
    "    cifar10_dm = CIFAR10DataModule(\n",
    "        data_dir=data_path,\n",
    "        batch_size=batch_size,\n",
    "        num_workers=num_workers,\n",
    "        train_transforms=train_transforms,\n",
    "        test_transforms=test_transforms,\n",
    "        val_transforms=test_transforms\n",
    "    )\n",
    "    return cifar10_dm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Resnet\n",
    "---\n",
    "Modify the pre-existing Resnet architecture from TorchVision. The pre-existing architecture is based on ImageNet images (224x224) as input. So we need to modify it for CIFAR10 images (32x32)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def create_model():\n",
    "    model = torchvision.models.resnet18(pretrained=False, num_classes=10)\n",
    "    model.conv1 = nn.Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
    "    model.maxpool = nn.Identity()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Lightning Module\n",
    "---\n",
    "Check out the [configure_optimizers](https://pytorch-lightning.readthedocs.io/en/stable/common/lightning_module.html#configure-optimizers) method to use custom Learning Rate schedulers. The OneCycleLR with SGD will get you to around 92-93% accuracy in 20-30 epochs and 93-94% accuracy in 40-50 epochs. Feel free to experiment with different LR schedules from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class LitResnet(LightningModule):\n",
    "    def __init__(self, learning_rate=0.05, steps_per_epoch=45000 , batch_size=32):\n",
    "        super().__init__()\n",
    "\n",
    "        self.save_hyperparameters()\n",
    "        self.model = create_model()\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = self.model(x)\n",
    "        return F.log_softmax(out, dim=1)\n",
    "\n",
    "    def training_step(self, batch, batch_idx):\n",
    "        x, y = batch\n",
    "        logits = self(x)\n",
    "        loss = F.nll_loss(logits, y)\n",
    "        self.log(\"train_loss\", loss)\n",
    "        return loss\n",
    "\n",
    "    def evaluate(self, batch, stage=None):\n",
    "        x, y = batch\n",
    "        logits = self(x)\n",
    "        loss = F.nll_loss(logits, y)\n",
    "        preds = torch.argmax(logits, dim=1)\n",
    "        acc = accuracy(preds, y, 'multiclass', num_classes=10)\n",
    "\n",
    "        if stage:\n",
    "            self.log(f\"{stage}_loss\", loss, prog_bar=True)\n",
    "            self.log(f\"{stage}_acc\", acc, prog_bar=True)\n",
    "\n",
    "    def validation_step(self, batch, batch_idx):\n",
    "        self.evaluate(batch, \"val\")\n",
    "\n",
    "    def test_step(self, batch, batch_idx):\n",
    "        self.evaluate(batch, \"test\")\n",
    "\n",
    "    def configure_optimizers(self):\n",
    "        optimizer = torch.optim.SGD(\n",
    "            self.parameters(),\n",
    "            lr=self.hparams.learning_rate,\n",
    "            momentum=0.9,\n",
    "            weight_decay=5e-4,\n",
    "        )\n",
    "        steps_per_epoch = self.hparams.steps_per_epoch // self.hparams.batch_size\n",
    "        scheduler_dict = {\n",
    "            \"scheduler\": OneCycleLR(\n",
    "                optimizer,\n",
    "                0.1,\n",
    "                epochs=self.trainer.max_epochs,\n",
    "                steps_per_epoch=steps_per_epoch,\n",
    "            ),\n",
    "            \"interval\": \"step\",\n",
    "        }\n",
    "        return {\"optimizer\": optimizer, \"lr_scheduler\": scheduler_dict}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 7\n"
     ]
    }
   ],
   "source": [
    "seed_everything(7)\n",
    "PATH_DATASETS = os.environ.get(\"PATH_DATASETS\", \".\")\n",
    "BATCH_SIZE = 64\n",
    "NUM_WORKERS = 0\n",
    "data_module = prepare_data(PATH_DATASETS, BATCH_SIZE, NUM_WORKERS)\n",
    "DEV_RUN = bool(os.environ.get('DEV_RUN', False))\n",
    "EPOCHS = 30"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Train\n",
    "Use Trainer from bigdl.nano.pytorch.trainer for BigDL-Nano pytorch.\n",
    "\n",
    "This Trainer extends PyTorch Lightning Trainer by adding various options to accelerate pytorch training.\n",
    "\n",
    "```\n",
    "    :param num_processes: number of processes in distributed training. default: 4.\n",
    "    :param use_ipex: whether we use ipex as accelerator for trainer. default: True.\n",
    "    :param cpu_for_each_process: A list of length `num_processes`, each containing a list of\n",
    "            indices of cpus each process will be using. default: None, and the cpu will be\n",
    "            automatically and evenly distributed among processes.\n",
    "```\n",
    "The next few cells show examples of different parameters.\n",
    "#### Single Process\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: False, used: False\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  | Name  | Type   | Params\n",
      "---------------------------------\n",
      "0 | model | ResNet | 11.2 M\n",
      "---------------------------------\n",
      "11.2 M    Trainable params\n",
      "0         Non-trainable params\n",
      "11.2 M    Total params\n",
      "44.696    Total estimated model params size (MB)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "69262a4e6eec45a583b19aea15995de6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation sanity check: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:106: UserWarning: The dataloader, val dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 96 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  f\"The dataloader, {name}, does not have many workers which may be a bottleneck.\"\n",
      "Global seed set to 7\n",
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:106: UserWarning: The dataloader, train dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 96 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  f\"The dataloader, {name}, does not have many workers which may be a bottleneck.\"\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6cec1437f97e4926b71da7d893f67aa3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: -1it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:106: UserWarning: The dataloader, test dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 96 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  f\"The dataloader, {name}, does not have many workers which may be a bottleneck.\"\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "551e2fc1950a4cdaa1c9ad9b18e9596f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Testing: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------------------------------------\n",
      "DATALOADER:0 TEST RESULTS\n",
      "{'test_acc': 0.902899980545044, 'test_loss': 0.2910485565662384}\n",
      "--------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "model = LitResnet(learning_rate=0.05)\n",
    "model.datamodule = data_module\n",
    "checkpoint_callback = ModelCheckpoint(dirpath=\"checkpoints/\", save_top_k=1, monitor=\"val_loss\", filename=\"renet18_single_none\")\n",
    "basic_trainer = Trainer(num_processes = 1,\n",
    "                  use_ipex = False,\n",
    "                  progress_bar_refresh_rate=10,\n",
    "                  max_epochs=EPOCHS,\n",
    "                  fast_dev_run=DEV_RUN,\n",
    "                  logger=TensorBoardLogger(\"lightning_logs/\", name=\"basic\"),\n",
    "                  callbacks=[LearningRateMonitor(logging_interval=\"step\"), checkpoint_callback])\n",
    "start = time()\n",
    "basic_trainer.fit(model, datamodule=data_module)\n",
    "basic_fit_time = time() - start\n",
    "outputs = basic_trainer.test(model, datamodule=data_module)\n",
    "basic_acc = outputs[0]['test_acc'] * 100\n",
    "basic_trainer.save_checkpoint(\"checkpoints/model.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Single Process with IPEX\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:446: UserWarning: Checkpoint directory checkpoints/ exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "GPU available: False, used: False\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/core/datamodule.py:424: LightningDeprecationWarning: DataModule.setup has already been called, so it will not be called again. In v1.6 this behavior will change to always call DataModule.setup.\n",
      "  f\"DataModule.{name} has already been called, so it will not be called again. \"\n",
      "\n",
      "  | Name  | Type   | Params\n",
      "---------------------------------\n",
      "0 | model | ResNet | 11.2 M\n",
      "---------------------------------\n",
      "11.2 M    Trainable params\n",
      "0         Non-trainable params\n",
      "11.2 M    Total params\n",
      "44.696    Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[WARNING] \"import intel_pytorch_extension\" will be deprecated in future releases. Please use \"import torch_ipex\" instead.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "482b2f7023414bcab3868736747c2931",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validation sanity check: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 7\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3cbea5d04c91436384465cc712834c8a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: -1it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[W LegacyTypeDispatch.h:79] Warning: AutoNonVariableTypeMode is deprecated and will be removed in 1.10 release. For kernel implementations please use AutoDispatchBelowADInplaceOrView instead, If you are looking for a user facing API to enable running your inference-only workload, please use c10::InferenceMode. Using AutoDispatchBelowADInplaceOrView in user code is under risk of producing silent wrong result in some edge cases. See Note [AutoDispatchBelowAutograd] for more details. (function operator())\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Validating: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/core/datamodule.py:424: LightningDeprecationWarning: DataModule.teardown has already been called, so it will not be called again. In v1.6 this behavior will change to always call DataModule.teardown.\n",
      "  f\"DataModule.{name} has already been called, so it will not be called again. \"\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14db67193dcd47e2bc1717477fd96512",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Testing: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------------------------------------\n",
      "DATALOADER:0 TEST RESULTS\n",
      "{'test_acc': 0.899399995803833, 'test_loss': 0.2945179343223572}\n",
      "--------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "model = LitResnet(learning_rate=0.05)\n",
    "model.datamodule = data_module\n",
    "checkpoint_callback = ModelCheckpoint(dirpath=\"checkpoints/\", save_top_k=1, monitor=\"val_loss\", filename=\"renet18_single_ipex\", save_weights_only=True)\n",
    "single_ipex_trainer = Trainer(num_processes=1,\n",
    "                        use_ipex = True,\n",
    "                        distributed_backend=\"subprocess\",\n",
    "                        progress_bar_refresh_rate=10,\n",
    "                        max_epochs=EPOCHS,\n",
    "                        fast_dev_run=DEV_RUN,\n",
    "                        logger=TensorBoardLogger(\"lightning_logs/\", name=\"single_ipex\"),\n",
    "                        callbacks=[LearningRateMonitor(logging_interval=\"step\"), checkpoint_callback])\n",
    "start = time()\n",
    "single_ipex_trainer.fit(model, datamodule=data_module)\n",
    "single_ipex_fit_time = time() - start\n",
    "outputs = single_ipex_trainer.test(model, datamodule=data_module)\n",
    "single_ipex_acc = outputs[0]['test_acc'] * 100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Multiple Processes with IPEX\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:446: UserWarning: Checkpoint directory checkpoints/ exists and is not empty.\n",
      "  rank_zero_warn(f\"Checkpoint directory {dirpath} exists and is not empty.\")\n",
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/plugins/training_type/ddp_spawn.py:75: LightningDeprecationWarning: Argument `num_nodes` in `DDPSpawnPlugin` is deprecated in v1.4, and will be removed in v1.6. Notice that it will be overriden by the trainer setting.\n",
      "  \"Argument `num_nodes` in `DDPSpawnPlugin` is deprecated in v1.4, and will be removed in v1.6. \"\n",
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/plugins/training_type/ddp_spawn.py:81: LightningDeprecationWarning: Argument `sync_batchnorm` in `DDPSpawnPlugin` is deprecated in v1.4, and will be removed in v1.6. Notice that it will be overriden by the trainer setting.\n",
      "  \"Argument `sync_batchnorm` in `DDPSpawnPlugin` is deprecated in v1.4, and will be removed in v1.6. \"\n",
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/trainer/connectors/accelerator_connector.py:685: UserWarning: Specified `Precision` and `TrainingType` plugins will be ignored, since an `Accelerator` instance was provided.\n",
      "  \"Specified `Precision` and `TrainingType` plugins will be ignored,\"\n",
      "GPU available: False, used: False\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/core/datamodule.py:424: LightningDeprecationWarning: DataModule.setup has already been called, so it will not be called again. In v1.6 this behavior will change to always call DataModule.setup.\n",
      "  f\"DataModule.{name} has already been called, so it will not be called again. \"\n",
      "Global seed set to 7\n",
      "initializing ddp: GLOBAL_RANK: 0, MEMBER: 1/2\n",
      "Global seed set to 7\n",
      "initializing ddp: GLOBAL_RANK: 1, MEMBER: 2/2\n",
      "----------------------------------------------------------------------------------------------------\n",
      "distributed_backend=gloo\n",
      "All DDP processes registered. Starting ddp with 2 processes\n",
      "----------------------------------------------------------------------------------------------------\n",
      "\n",
      "\n",
      "  | Name  | Type   | Params\n",
      "---------------------------------\n",
      "0 | model | ResNet | 11.2 M\n",
      "---------------------------------\n",
      "11.2 M    Trainable params\n",
      "0         Non-trainable params\n",
      "11.2 M    Total params\n",
      "44.696    Total estimated model params size (MB)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[WARNING] \"import intel_pytorch_extension\" will be deprecated in future releases. Please use \"import torch_ipex\" instead.\n",
      "Epoch 0:   0%|          | 0/782 [00:00<00:00, 3953.16it/s]    "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:106: UserWarning: The dataloader, val dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 96 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  f\"The dataloader, {name}, does not have many workers which may be a bottleneck.\"\n",
      "Global seed set to 7\n",
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:106: UserWarning: The dataloader, val dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 96 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  f\"The dataloader, {name}, does not have many workers which may be a bottleneck.\"\n",
      "Global seed set to 7\n",
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:106: UserWarning: The dataloader, train dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 96 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  f\"The dataloader, {name}, does not have many workers which may be a bottleneck.\"\n",
      "[W LegacyTypeDispatch.h:79] Warning: AutoNonVariableTypeMode is deprecated and will be removed in 1.10 release. For kernel implementations please use AutoDispatchBelowADInplaceOrView instead, If you are looking for a user facing API to enable running your inference-only workload, please use c10::InferenceMode. Using AutoDispatchBelowADInplaceOrView in user code is under risk of producing silent wrong result in some edge cases. See Note [AutoDispatchBelowAutograd] for more details. (function operator())\n",
      "[W LegacyTypeDispatch.h:79] Warning: AutoNonVariableTypeMode is deprecated and will be removed in 1.10 release. For kernel implementations please use AutoDispatchBelowADInplaceOrView instead, If you are looking for a user facing API to enable running your inference-only workload, please use c10::InferenceMode. Using AutoDispatchBelowADInplaceOrView in user code is under risk of producing silent wrong result in some edge cases. See Note [AutoDispatchBelowAutograd] for more details. (function operator())\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0:  81%|████████  | 630/782 [01:31<00:21,  6.92it/s, loss=1.54, v_num=1]\n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:125: UserWarning: Seems like `optimizer.step()` has been overridden after learning rate scheduler initialization. Please, make sure to call `optimizer.step()` before `lr_scheduler.step()`. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
      "  \"https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\", UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 0:  82%|████████▏ | 640/782 [01:31<00:20,  7.00it/s, loss=1.54, v_num=1]\n",
      "Epoch 0:  83%|████████▎ | 650/782 [01:31<00:18,  7.08it/s, loss=1.54, v_num=1]\n",
      "Epoch 0:  84%|████████▍ | 660/782 [01:32<00:17,  7.16it/s, loss=1.54, v_num=1]\n",
      "Epoch 0:  86%|████████▌ | 670/782 [01:32<00:15,  7.24it/s, loss=1.54, v_num=1]\n",
      "Epoch 0:  87%|████████▋ | 680/782 [01:33<00:13,  7.32it/s, loss=1.54, v_num=1]\n",
      "Epoch 0:  88%|████████▊ | 690/782 [01:33<00:12,  7.40it/s, loss=1.54, v_num=1]\n",
      "Epoch 0:  90%|████████▉ | 700/782 [01:33<00:10,  7.47it/s, loss=1.54, v_num=1]\n",
      "Epoch 0:  91%|█████████ | 710/782 [01:34<00:09,  7.56it/s, loss=1.54, v_num=1]\n",
      "Epoch 0:  92%|█████████▏| 720/782 [01:34<00:08,  7.63it/s, loss=1.54, v_num=1]\n",
      "Epoch 0:  93%|█████████▎| 730/782 [01:34<00:06,  7.71it/s, loss=1.54, v_num=1]\n",
      "Epoch 0:  95%|█████████▍| 740/782 [01:35<00:05,  7.78it/s, loss=1.54, v_num=1]\n",
      "Epoch 0:  96%|█████████▌| 750/782 [01:35<00:04,  7.86it/s, loss=1.54, v_num=1]\n",
      "Epoch 0:  97%|█████████▋| 760/782 [01:35<00:02,  7.94it/s, loss=1.54, v_num=1]\n",
      "Epoch 0:  98%|█████████▊| 770/782 [01:36<00:01,  8.02it/s, loss=1.54, v_num=1]\n",
      "Epoch 0: 100%|█████████▉| 780/782 [01:36<00:00,  8.09it/s, loss=1.54, v_num=1]\n",
      "Epoch 0: 100%|██████████| 782/782 [01:36<00:00,  8.09it/s, loss=1.54, v_num=1, val_loss=1.510, val_acc=0.456]\n",
      "Epoch 1:  81%|████████  | 630/782 [01:39<00:24,  6.32it/s, loss=1.47, v_num=1, val_loss=1.510, val_acc=0.456] \n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 1:  82%|████████▏ | 640/782 [01:40<00:22,  6.40it/s, loss=1.47, v_num=1, val_loss=1.510, val_acc=0.456]\n",
      "Epoch 1:  83%|████████▎ | 650/782 [01:40<00:20,  6.48it/s, loss=1.47, v_num=1, val_loss=1.510, val_acc=0.456]\n",
      "Epoch 1:  84%|████████▍ | 660/782 [01:40<00:18,  6.56it/s, loss=1.47, v_num=1, val_loss=1.510, val_acc=0.456]\n",
      "Epoch 1:  86%|████████▌ | 670/782 [01:41<00:16,  6.63it/s, loss=1.47, v_num=1, val_loss=1.510, val_acc=0.456]\n",
      "Epoch 1:  87%|████████▋ | 680/782 [01:41<00:15,  6.71it/s, loss=1.47, v_num=1, val_loss=1.510, val_acc=0.456]\n",
      "Epoch 1:  88%|████████▊ | 690/782 [01:41<00:13,  6.79it/s, loss=1.47, v_num=1, val_loss=1.510, val_acc=0.456]\n",
      "Epoch 1:  90%|████████▉ | 700/782 [01:42<00:11,  6.87it/s, loss=1.47, v_num=1, val_loss=1.510, val_acc=0.456]\n",
      "Epoch 1:  91%|█████████ | 710/782 [01:42<00:10,  6.94it/s, loss=1.47, v_num=1, val_loss=1.510, val_acc=0.456]\n",
      "Epoch 1:  92%|█████████▏| 720/782 [01:42<00:08,  7.01it/s, loss=1.47, v_num=1, val_loss=1.510, val_acc=0.456]\n",
      "Epoch 1:  93%|█████████▎| 730/782 [01:43<00:07,  7.09it/s, loss=1.47, v_num=1, val_loss=1.510, val_acc=0.456]\n",
      "Epoch 1:  95%|█████████▍| 740/782 [01:43<00:05,  7.16it/s, loss=1.47, v_num=1, val_loss=1.510, val_acc=0.456]\n",
      "Epoch 1:  96%|█████████▌| 750/782 [01:43<00:04,  7.23it/s, loss=1.47, v_num=1, val_loss=1.510, val_acc=0.456]\n",
      "Epoch 1:  97%|█████████▋| 760/782 [01:44<00:03,  7.31it/s, loss=1.47, v_num=1, val_loss=1.510, val_acc=0.456]\n",
      "Epoch 1:  98%|█████████▊| 770/782 [01:44<00:01,  7.38it/s, loss=1.47, v_num=1, val_loss=1.510, val_acc=0.456]\n",
      "Epoch 1: 100%|█████████▉| 780/782 [01:44<00:00,  7.45it/s, loss=1.47, v_num=1, val_loss=1.510, val_acc=0.456]\n",
      "Epoch 1: 100%|██████████| 782/782 [01:45<00:00,  7.45it/s, loss=1.53, v_num=1, val_loss=1.470, val_acc=0.497]\n",
      "Epoch 2:  81%|████████  | 630/782 [01:18<00:18,  8.02it/s, loss=1.14, v_num=1, val_loss=1.470, val_acc=0.497]\n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 2:  82%|████████▏ | 640/782 [01:19<00:17,  8.11it/s, loss=1.14, v_num=1, val_loss=1.470, val_acc=0.497]\n",
      "Epoch 2:  83%|████████▎ | 650/782 [01:19<00:16,  8.21it/s, loss=1.14, v_num=1, val_loss=1.470, val_acc=0.497]\n",
      "Epoch 2:  84%|████████▍ | 660/782 [01:19<00:14,  8.29it/s, loss=1.14, v_num=1, val_loss=1.470, val_acc=0.497]\n",
      "Epoch 2:  86%|████████▌ | 670/782 [01:20<00:13,  8.38it/s, loss=1.14, v_num=1, val_loss=1.470, val_acc=0.497]\n",
      "Epoch 2:  87%|████████▋ | 680/782 [01:20<00:12,  8.46it/s, loss=1.14, v_num=1, val_loss=1.470, val_acc=0.497]\n",
      "Epoch 2:  88%|████████▊ | 690/782 [01:20<00:10,  8.55it/s, loss=1.14, v_num=1, val_loss=1.470, val_acc=0.497]\n",
      "Epoch 2:  90%|████████▉ | 700/782 [01:21<00:09,  8.64it/s, loss=1.14, v_num=1, val_loss=1.470, val_acc=0.497]\n",
      "Epoch 2:  91%|█████████ | 710/782 [01:21<00:08,  8.73it/s, loss=1.14, v_num=1, val_loss=1.470, val_acc=0.497]\n",
      "Epoch 2:  92%|█████████▏| 720/782 [01:21<00:07,  8.81it/s, loss=1.14, v_num=1, val_loss=1.470, val_acc=0.497]\n",
      "Epoch 2:  93%|█████████▎| 730/782 [01:22<00:05,  8.89it/s, loss=1.14, v_num=1, val_loss=1.470, val_acc=0.497]\n",
      "Epoch 2:  95%|█████████▍| 740/782 [01:22<00:04,  8.97it/s, loss=1.14, v_num=1, val_loss=1.470, val_acc=0.497]\n",
      "Epoch 2:  96%|█████████▌| 750/782 [01:23<00:03,  9.04it/s, loss=1.14, v_num=1, val_loss=1.470, val_acc=0.497]\n",
      "Epoch 2:  97%|█████████▋| 760/782 [01:23<00:02,  9.12it/s, loss=1.14, v_num=1, val_loss=1.470, val_acc=0.497]\n",
      "Epoch 2:  98%|█████████▊| 770/782 [01:23<00:01,  9.20it/s, loss=1.14, v_num=1, val_loss=1.470, val_acc=0.497]\n",
      "Epoch 2: 100%|█████████▉| 780/782 [01:24<00:00,  9.28it/s, loss=1.14, v_num=1, val_loss=1.470, val_acc=0.497]\n",
      "Epoch 2: 100%|██████████| 782/782 [01:24<00:00,  9.26it/s, loss=1.19, v_num=1, val_loss=1.280, val_acc=0.567]\n",
      "Epoch 3:  81%|████████  | 630/782 [01:12<00:17,  8.70it/s, loss=1.05, v_num=1, val_loss=1.280, val_acc=0.567] \n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 3:  82%|████████▏ | 640/782 [01:12<00:16,  8.80it/s, loss=1.05, v_num=1, val_loss=1.280, val_acc=0.567]\n",
      "Epoch 3:  83%|████████▎ | 650/782 [01:13<00:14,  8.90it/s, loss=1.05, v_num=1, val_loss=1.280, val_acc=0.567]\n",
      "Epoch 3:  84%|████████▍ | 660/782 [01:13<00:13,  8.99it/s, loss=1.05, v_num=1, val_loss=1.280, val_acc=0.567]\n",
      "Epoch 3:  86%|████████▌ | 670/782 [01:13<00:12,  9.08it/s, loss=1.05, v_num=1, val_loss=1.280, val_acc=0.567]\n",
      "Epoch 3:  87%|████████▋ | 680/782 [01:14<00:11,  9.18it/s, loss=1.05, v_num=1, val_loss=1.280, val_acc=0.567]\n",
      "Epoch 3:  88%|████████▊ | 690/782 [01:14<00:09,  9.27it/s, loss=1.05, v_num=1, val_loss=1.280, val_acc=0.567]\n",
      "Epoch 3:  90%|████████▉ | 700/782 [01:14<00:08,  9.35it/s, loss=1.05, v_num=1, val_loss=1.280, val_acc=0.567]\n",
      "Epoch 3:  91%|█████████ | 710/782 [01:15<00:07,  9.44it/s, loss=1.05, v_num=1, val_loss=1.280, val_acc=0.567]\n",
      "Epoch 3:  92%|█████████▏| 720/782 [01:15<00:06,  9.53it/s, loss=1.05, v_num=1, val_loss=1.280, val_acc=0.567]\n",
      "Epoch 3:  93%|█████████▎| 730/782 [01:16<00:05,  9.61it/s, loss=1.05, v_num=1, val_loss=1.280, val_acc=0.567]\n",
      "Epoch 3:  95%|█████████▍| 740/782 [01:16<00:04,  9.69it/s, loss=1.05, v_num=1, val_loss=1.280, val_acc=0.567]\n",
      "Epoch 3:  96%|█████████▌| 750/782 [01:16<00:03,  9.77it/s, loss=1.05, v_num=1, val_loss=1.280, val_acc=0.567]\n",
      "Epoch 3:  97%|█████████▋| 760/782 [01:17<00:02,  9.85it/s, loss=1.05, v_num=1, val_loss=1.280, val_acc=0.567]\n",
      "Epoch 3:  98%|█████████▊| 770/782 [01:17<00:01,  9.92it/s, loss=1.05, v_num=1, val_loss=1.280, val_acc=0.567]\n",
      "Epoch 3: 100%|█████████▉| 780/782 [01:18<00:00,  9.99it/s, loss=1.05, v_num=1, val_loss=1.280, val_acc=0.567]\n",
      "Epoch 3: 100%|██████████| 782/782 [01:18<00:00,  9.94it/s, loss=1.08, v_num=1, val_loss=1.100, val_acc=0.620]\n",
      "Epoch 4:  81%|████████  | 630/782 [01:16<00:18,  8.27it/s, loss=0.993, v_num=1, val_loss=1.100, val_acc=0.620]\n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 4:  82%|████████▏ | 640/782 [01:16<00:16,  8.37it/s, loss=0.993, v_num=1, val_loss=1.100, val_acc=0.620]\n",
      "Epoch 4:  83%|████████▎ | 650/782 [01:16<00:15,  8.47it/s, loss=0.993, v_num=1, val_loss=1.100, val_acc=0.620]\n",
      "Epoch 4:  84%|████████▍ | 660/782 [01:17<00:14,  8.55it/s, loss=0.993, v_num=1, val_loss=1.100, val_acc=0.620]\n",
      "Epoch 4:  86%|████████▌ | 670/782 [01:17<00:12,  8.64it/s, loss=0.993, v_num=1, val_loss=1.100, val_acc=0.620]\n",
      "Epoch 4:  87%|████████▋ | 680/782 [01:18<00:11,  8.73it/s, loss=0.993, v_num=1, val_loss=1.100, val_acc=0.620]\n",
      "Epoch 4:  88%|████████▊ | 690/782 [01:18<00:10,  8.81it/s, loss=0.993, v_num=1, val_loss=1.100, val_acc=0.620]\n",
      "Epoch 4:  90%|████████▉ | 700/782 [01:18<00:09,  8.90it/s, loss=0.993, v_num=1, val_loss=1.100, val_acc=0.620]\n",
      "Epoch 4:  91%|█████████ | 710/782 [01:19<00:08,  8.99it/s, loss=0.993, v_num=1, val_loss=1.100, val_acc=0.620]\n",
      "Epoch 4:  92%|█████████▏| 720/782 [01:19<00:06,  9.08it/s, loss=0.993, v_num=1, val_loss=1.100, val_acc=0.620]\n",
      "Epoch 4:  93%|█████████▎| 730/782 [01:19<00:05,  9.17it/s, loss=0.993, v_num=1, val_loss=1.100, val_acc=0.620]\n",
      "Epoch 4:  95%|█████████▍| 740/782 [01:20<00:04,  9.26it/s, loss=0.993, v_num=1, val_loss=1.100, val_acc=0.620]\n",
      "Epoch 4:  96%|█████████▌| 750/782 [01:20<00:03,  9.34it/s, loss=0.993, v_num=1, val_loss=1.100, val_acc=0.620]\n",
      "Epoch 4:  97%|█████████▋| 760/782 [01:20<00:02,  9.43it/s, loss=0.993, v_num=1, val_loss=1.100, val_acc=0.620]\n",
      "Epoch 4:  98%|█████████▊| 770/782 [01:21<00:01,  9.52it/s, loss=0.993, v_num=1, val_loss=1.100, val_acc=0.620]\n",
      "Epoch 4: 100%|█████████▉| 780/782 [01:21<00:00,  9.60it/s, loss=0.993, v_num=1, val_loss=1.100, val_acc=0.620]\n",
      "Epoch 4: 100%|██████████| 782/782 [01:21<00:00,  9.58it/s, loss=1.04, v_num=1, val_loss=0.992, val_acc=0.648] \n",
      "Epoch 5:  81%|████████  | 630/782 [01:13<00:17,  8.58it/s, loss=0.764, v_num=1, val_loss=0.992, val_acc=0.648]\n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 5:  82%|████████▏ | 640/782 [01:13<00:16,  8.67it/s, loss=0.764, v_num=1, val_loss=0.992, val_acc=0.648]\n",
      "Epoch 5:  83%|████████▎ | 650/782 [01:14<00:15,  8.77it/s, loss=0.764, v_num=1, val_loss=0.992, val_acc=0.648]\n",
      "Epoch 5:  84%|████████▍ | 660/782 [01:14<00:13,  8.86it/s, loss=0.764, v_num=1, val_loss=0.992, val_acc=0.648]\n",
      "Epoch 5:  86%|████████▌ | 670/782 [01:14<00:12,  8.96it/s, loss=0.764, v_num=1, val_loss=0.992, val_acc=0.648]\n",
      "Epoch 5:  87%|████████▋ | 680/782 [01:15<00:11,  9.06it/s, loss=0.764, v_num=1, val_loss=0.992, val_acc=0.648]\n",
      "Epoch 5:  88%|████████▊ | 690/782 [01:15<00:10,  9.15it/s, loss=0.764, v_num=1, val_loss=0.992, val_acc=0.648]\n",
      "Epoch 5:  90%|████████▉ | 700/782 [01:15<00:08,  9.24it/s, loss=0.764, v_num=1, val_loss=0.992, val_acc=0.648]\n",
      "Epoch 5:  91%|█████████ | 710/782 [01:16<00:07,  9.34it/s, loss=0.764, v_num=1, val_loss=0.992, val_acc=0.648]\n",
      "Epoch 5:  92%|█████████▏| 720/782 [01:16<00:06,  9.43it/s, loss=0.764, v_num=1, val_loss=0.992, val_acc=0.648]\n",
      "Epoch 5:  93%|█████████▎| 730/782 [01:16<00:05,  9.52it/s, loss=0.764, v_num=1, val_loss=0.992, val_acc=0.648]\n",
      "Epoch 5:  95%|█████████▍| 740/782 [01:17<00:04,  9.61it/s, loss=0.764, v_num=1, val_loss=0.992, val_acc=0.648]\n",
      "Epoch 5:  96%|█████████▌| 750/782 [01:17<00:03,  9.70it/s, loss=0.764, v_num=1, val_loss=0.992, val_acc=0.648]\n",
      "Epoch 5:  97%|█████████▋| 760/782 [01:17<00:02,  9.79it/s, loss=0.764, v_num=1, val_loss=0.992, val_acc=0.648]\n",
      "Epoch 5:  98%|█████████▊| 770/782 [01:18<00:01,  9.88it/s, loss=0.764, v_num=1, val_loss=0.992, val_acc=0.648]\n",
      "Epoch 5: 100%|█████████▉| 780/782 [01:18<00:00,  9.97it/s, loss=0.764, v_num=1, val_loss=0.992, val_acc=0.648]\n",
      "Epoch 5: 100%|██████████| 782/782 [01:18<00:00,  9.96it/s, loss=0.839, v_num=1, val_loss=1.330, val_acc=0.587]\n",
      "Epoch 6:  81%|████████  | 630/782 [01:43<00:25,  6.07it/s, loss=0.901, v_num=1, val_loss=1.330, val_acc=0.587] \n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 6:  82%|████████▏ | 640/782 [01:44<00:23,  6.15it/s, loss=0.901, v_num=1, val_loss=1.330, val_acc=0.587]\n",
      "Epoch 6:  83%|████████▎ | 650/782 [01:44<00:21,  6.22it/s, loss=0.901, v_num=1, val_loss=1.330, val_acc=0.587]\n",
      "Epoch 6:  84%|████████▍ | 660/782 [01:44<00:19,  6.30it/s, loss=0.901, v_num=1, val_loss=1.330, val_acc=0.587]\n",
      "Epoch 6:  86%|████████▌ | 670/782 [01:45<00:17,  6.38it/s, loss=0.901, v_num=1, val_loss=1.330, val_acc=0.587]\n",
      "Epoch 6:  87%|████████▋ | 680/782 [01:45<00:15,  6.45it/s, loss=0.901, v_num=1, val_loss=1.330, val_acc=0.587]\n",
      "Epoch 6:  88%|████████▊ | 690/782 [01:45<00:14,  6.53it/s, loss=0.901, v_num=1, val_loss=1.330, val_acc=0.587]\n",
      "Epoch 6:  90%|████████▉ | 700/782 [01:46<00:12,  6.60it/s, loss=0.901, v_num=1, val_loss=1.330, val_acc=0.587]\n",
      "Epoch 6:  91%|█████████ | 710/782 [01:46<00:10,  6.68it/s, loss=0.901, v_num=1, val_loss=1.330, val_acc=0.587]\n",
      "Epoch 6:  92%|█████████▏| 720/782 [01:46<00:09,  6.75it/s, loss=0.901, v_num=1, val_loss=1.330, val_acc=0.587]\n",
      "Epoch 6:  93%|█████████▎| 730/782 [01:47<00:07,  6.83it/s, loss=0.901, v_num=1, val_loss=1.330, val_acc=0.587]\n",
      "Epoch 6:  95%|█████████▍| 740/782 [01:47<00:06,  6.90it/s, loss=0.901, v_num=1, val_loss=1.330, val_acc=0.587]\n",
      "Epoch 6:  96%|█████████▌| 750/782 [01:47<00:04,  6.97it/s, loss=0.901, v_num=1, val_loss=1.330, val_acc=0.587]\n",
      "Epoch 6:  97%|█████████▋| 760/782 [01:48<00:03,  7.04it/s, loss=0.901, v_num=1, val_loss=1.330, val_acc=0.587]\n",
      "Epoch 6:  98%|█████████▊| 770/782 [01:48<00:01,  7.12it/s, loss=0.901, v_num=1, val_loss=1.330, val_acc=0.587]\n",
      "Epoch 6: 100%|█████████▉| 780/782 [01:48<00:00,  7.19it/s, loss=0.901, v_num=1, val_loss=1.330, val_acc=0.587]\n",
      "Epoch 6: 100%|██████████| 782/782 [01:48<00:00,  7.19it/s, loss=0.924, v_num=1, val_loss=0.882, val_acc=0.695]\n",
      "Epoch 7:  81%|████████  | 630/782 [01:29<00:21,  7.08it/s, loss=0.718, v_num=1, val_loss=0.882, val_acc=0.695] \n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 7:  82%|████████▏ | 640/782 [01:29<00:19,  7.17it/s, loss=0.718, v_num=1, val_loss=0.882, val_acc=0.695]\n",
      "Epoch 7:  83%|████████▎ | 650/782 [01:29<00:18,  7.26it/s, loss=0.718, v_num=1, val_loss=0.882, val_acc=0.695]\n",
      "Epoch 7:  84%|████████▍ | 660/782 [01:30<00:16,  7.34it/s, loss=0.718, v_num=1, val_loss=0.882, val_acc=0.695]\n",
      "Epoch 7:  86%|████████▌ | 670/782 [01:30<00:15,  7.43it/s, loss=0.718, v_num=1, val_loss=0.882, val_acc=0.695]\n",
      "Epoch 7:  87%|████████▋ | 680/782 [01:30<00:13,  7.51it/s, loss=0.718, v_num=1, val_loss=0.882, val_acc=0.695]\n",
      "Epoch 7:  88%|████████▊ | 690/782 [01:30<00:12,  7.60it/s, loss=0.718, v_num=1, val_loss=0.882, val_acc=0.695]\n",
      "Epoch 7:  90%|████████▉ | 700/782 [01:31<00:10,  7.68it/s, loss=0.718, v_num=1, val_loss=0.882, val_acc=0.695]\n",
      "Epoch 7:  91%|█████████ | 710/782 [01:31<00:09,  7.76it/s, loss=0.718, v_num=1, val_loss=0.882, val_acc=0.695]\n",
      "Epoch 7:  92%|█████████▏| 720/782 [01:31<00:07,  7.84it/s, loss=0.718, v_num=1, val_loss=0.882, val_acc=0.695]\n",
      "Epoch 7:  93%|█████████▎| 730/782 [01:32<00:06,  7.93it/s, loss=0.718, v_num=1, val_loss=0.882, val_acc=0.695]\n",
      "Epoch 7:  95%|█████████▍| 740/782 [01:32<00:05,  8.01it/s, loss=0.718, v_num=1, val_loss=0.882, val_acc=0.695]\n",
      "Epoch 7:  96%|█████████▌| 750/782 [01:32<00:03,  8.09it/s, loss=0.718, v_num=1, val_loss=0.882, val_acc=0.695]\n",
      "Epoch 7:  97%|█████████▋| 760/782 [01:33<00:02,  8.16it/s, loss=0.718, v_num=1, val_loss=0.882, val_acc=0.695]\n",
      "Epoch 7:  98%|█████████▊| 770/782 [01:33<00:01,  8.24it/s, loss=0.718, v_num=1, val_loss=0.882, val_acc=0.695]\n",
      "Epoch 7: 100%|█████████▉| 780/782 [01:33<00:00,  8.32it/s, loss=0.718, v_num=1, val_loss=0.882, val_acc=0.695]\n",
      "Epoch 7: 100%|██████████| 782/782 [01:34<00:00,  8.31it/s, loss=0.668, v_num=1, val_loss=0.802, val_acc=0.735]\n",
      "Epoch 8:  81%|████████  | 630/782 [01:16<00:18,  8.29it/s, loss=0.738, v_num=1, val_loss=0.802, val_acc=0.735] \n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 8:  82%|████████▏ | 640/782 [01:16<00:16,  8.39it/s, loss=0.738, v_num=1, val_loss=0.802, val_acc=0.735]\n",
      "Epoch 8:  83%|████████▎ | 650/782 [01:16<00:15,  8.48it/s, loss=0.738, v_num=1, val_loss=0.802, val_acc=0.735]\n",
      "Epoch 8:  84%|████████▍ | 660/782 [01:17<00:14,  8.58it/s, loss=0.738, v_num=1, val_loss=0.802, val_acc=0.735]\n",
      "Epoch 8:  86%|████████▌ | 670/782 [01:17<00:12,  8.67it/s, loss=0.738, v_num=1, val_loss=0.802, val_acc=0.735]\n",
      "Epoch 8:  87%|████████▋ | 680/782 [01:17<00:11,  8.76it/s, loss=0.738, v_num=1, val_loss=0.802, val_acc=0.735]\n",
      "Epoch 8:  88%|████████▊ | 690/782 [01:18<00:10,  8.85it/s, loss=0.738, v_num=1, val_loss=0.802, val_acc=0.735]\n",
      "Epoch 8:  90%|████████▉ | 700/782 [01:18<00:09,  8.94it/s, loss=0.738, v_num=1, val_loss=0.802, val_acc=0.735]\n",
      "Epoch 8:  91%|█████████ | 710/782 [01:18<00:07,  9.02it/s, loss=0.738, v_num=1, val_loss=0.802, val_acc=0.735]\n",
      "Epoch 8:  92%|█████████▏| 720/782 [01:19<00:06,  9.10it/s, loss=0.738, v_num=1, val_loss=0.802, val_acc=0.735]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8:  93%|█████████▎| 730/782 [01:19<00:05,  9.18it/s, loss=0.738, v_num=1, val_loss=0.802, val_acc=0.735]\n",
      "Epoch 8:  95%|█████████▍| 740/782 [01:20<00:04,  9.25it/s, loss=0.738, v_num=1, val_loss=0.802, val_acc=0.735]\n",
      "Epoch 8:  96%|█████████▌| 750/782 [01:20<00:03,  9.34it/s, loss=0.738, v_num=1, val_loss=0.802, val_acc=0.735]\n",
      "Epoch 8:  97%|█████████▋| 760/782 [01:20<00:02,  9.41it/s, loss=0.738, v_num=1, val_loss=0.802, val_acc=0.735]\n",
      "Epoch 8:  98%|█████████▊| 770/782 [01:21<00:01,  9.49it/s, loss=0.738, v_num=1, val_loss=0.802, val_acc=0.735]\n",
      "Epoch 8: 100%|█████████▉| 780/782 [01:21<00:00,  9.57it/s, loss=0.738, v_num=1, val_loss=0.802, val_acc=0.735]\n",
      "Epoch 8: 100%|██████████| 782/782 [01:22<00:00,  9.54it/s, loss=0.705, v_num=1, val_loss=0.726, val_acc=0.757]\n",
      "Epoch 9:  81%|████████  | 630/782 [01:19<00:19,  7.97it/s, loss=0.757, v_num=1, val_loss=0.726, val_acc=0.757] \n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 9:  82%|████████▏ | 640/782 [01:19<00:17,  8.05it/s, loss=0.757, v_num=1, val_loss=0.726, val_acc=0.757]\n",
      "Epoch 9:  83%|████████▎ | 650/782 [01:19<00:16,  8.14it/s, loss=0.757, v_num=1, val_loss=0.726, val_acc=0.757]\n",
      "Epoch 9:  84%|████████▍ | 660/782 [01:20<00:14,  8.23it/s, loss=0.757, v_num=1, val_loss=0.726, val_acc=0.757]\n",
      "Epoch 9:  86%|████████▌ | 670/782 [01:20<00:13,  8.31it/s, loss=0.757, v_num=1, val_loss=0.726, val_acc=0.757]\n",
      "Epoch 9:  87%|████████▋ | 680/782 [01:21<00:12,  8.39it/s, loss=0.757, v_num=1, val_loss=0.726, val_acc=0.757]\n",
      "Epoch 9:  88%|████████▊ | 690/782 [01:21<00:10,  8.47it/s, loss=0.757, v_num=1, val_loss=0.726, val_acc=0.757]\n",
      "Epoch 9:  90%|████████▉ | 700/782 [01:21<00:09,  8.56it/s, loss=0.757, v_num=1, val_loss=0.726, val_acc=0.757]\n",
      "Epoch 9:  91%|█████████ | 710/782 [01:22<00:08,  8.64it/s, loss=0.757, v_num=1, val_loss=0.726, val_acc=0.757]\n",
      "Epoch 9:  92%|█████████▏| 720/782 [01:22<00:07,  8.71it/s, loss=0.757, v_num=1, val_loss=0.726, val_acc=0.757]\n",
      "Epoch 9:  93%|█████████▎| 730/782 [01:23<00:05,  8.78it/s, loss=0.757, v_num=1, val_loss=0.726, val_acc=0.757]\n",
      "Epoch 9:  95%|█████████▍| 740/782 [01:23<00:04,  8.86it/s, loss=0.757, v_num=1, val_loss=0.726, val_acc=0.757]\n",
      "Epoch 9:  96%|█████████▌| 750/782 [01:23<00:03,  8.95it/s, loss=0.757, v_num=1, val_loss=0.726, val_acc=0.757]\n",
      "Epoch 9:  97%|█████████▋| 760/782 [01:24<00:02,  9.02it/s, loss=0.757, v_num=1, val_loss=0.726, val_acc=0.757]\n",
      "Epoch 9:  98%|█████████▊| 770/782 [01:24<00:01,  9.09it/s, loss=0.757, v_num=1, val_loss=0.726, val_acc=0.757]\n",
      "Epoch 9: 100%|█████████▉| 780/782 [01:25<00:00,  9.18it/s, loss=0.757, v_num=1, val_loss=0.726, val_acc=0.757]\n",
      "Epoch 9: 100%|██████████| 782/782 [01:25<00:00,  9.18it/s, loss=0.771, v_num=1, val_loss=0.912, val_acc=0.693]\n",
      "Epoch 10:  81%|████████  | 630/782 [01:29<00:21,  7.02it/s, loss=0.675, v_num=1, val_loss=0.912, val_acc=0.693]\n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 10:  82%|████████▏ | 640/782 [01:30<00:19,  7.10it/s, loss=0.675, v_num=1, val_loss=0.912, val_acc=0.693]\n",
      "Epoch 10:  83%|████████▎ | 650/782 [01:30<00:18,  7.18it/s, loss=0.675, v_num=1, val_loss=0.912, val_acc=0.693]\n",
      "Epoch 10:  84%|████████▍ | 660/782 [01:31<00:16,  7.26it/s, loss=0.675, v_num=1, val_loss=0.912, val_acc=0.693]\n",
      "Epoch 10:  86%|████████▌ | 670/782 [01:31<00:15,  7.33it/s, loss=0.675, v_num=1, val_loss=0.912, val_acc=0.693]\n",
      "Epoch 10:  87%|████████▋ | 680/782 [01:31<00:13,  7.41it/s, loss=0.675, v_num=1, val_loss=0.912, val_acc=0.693]\n",
      "Epoch 10:  88%|████████▊ | 690/782 [01:32<00:12,  7.49it/s, loss=0.675, v_num=1, val_loss=0.912, val_acc=0.693]\n",
      "Epoch 10:  90%|████████▉ | 700/782 [01:32<00:10,  7.57it/s, loss=0.675, v_num=1, val_loss=0.912, val_acc=0.693]\n",
      "Epoch 10:  91%|█████████ | 710/782 [01:33<00:09,  7.64it/s, loss=0.675, v_num=1, val_loss=0.912, val_acc=0.693]\n",
      "Epoch 10:  92%|█████████▏| 720/782 [01:33<00:08,  7.72it/s, loss=0.675, v_num=1, val_loss=0.912, val_acc=0.693]\n",
      "Epoch 10:  93%|█████████▎| 730/782 [01:33<00:06,  7.80it/s, loss=0.675, v_num=1, val_loss=0.912, val_acc=0.693]\n",
      "Epoch 10:  95%|█████████▍| 740/782 [01:34<00:05,  7.88it/s, loss=0.675, v_num=1, val_loss=0.912, val_acc=0.693]\n",
      "Epoch 10:  96%|█████████▌| 750/782 [01:34<00:04,  7.95it/s, loss=0.675, v_num=1, val_loss=0.912, val_acc=0.693]\n",
      "Epoch 10:  97%|█████████▋| 760/782 [01:34<00:02,  8.03it/s, loss=0.675, v_num=1, val_loss=0.912, val_acc=0.693]\n",
      "Epoch 10:  98%|█████████▊| 770/782 [01:35<00:01,  8.10it/s, loss=0.675, v_num=1, val_loss=0.912, val_acc=0.693]\n",
      "Epoch 10: 100%|█████████▉| 780/782 [01:35<00:00,  8.18it/s, loss=0.675, v_num=1, val_loss=0.912, val_acc=0.693]\n",
      "Epoch 10: 100%|██████████| 782/782 [01:36<00:00,  8.14it/s, loss=0.674, v_num=1, val_loss=0.716, val_acc=0.751]\n",
      "Epoch 11:  81%|████████  | 630/782 [01:16<00:18,  8.24it/s, loss=0.751, v_num=1, val_loss=0.716, val_acc=0.751]\n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 11:  82%|████████▏ | 640/782 [01:16<00:17,  8.34it/s, loss=0.751, v_num=1, val_loss=0.716, val_acc=0.751]\n",
      "Epoch 11:  83%|████████▎ | 650/782 [01:17<00:15,  8.43it/s, loss=0.751, v_num=1, val_loss=0.716, val_acc=0.751]\n",
      "Epoch 11:  84%|████████▍ | 660/782 [01:17<00:14,  8.53it/s, loss=0.751, v_num=1, val_loss=0.716, val_acc=0.751]\n",
      "Epoch 11:  86%|████████▌ | 670/782 [01:17<00:12,  8.62it/s, loss=0.751, v_num=1, val_loss=0.716, val_acc=0.751]\n",
      "Epoch 11:  87%|████████▋ | 680/782 [01:18<00:11,  8.71it/s, loss=0.751, v_num=1, val_loss=0.716, val_acc=0.751]\n",
      "Epoch 11:  88%|████████▊ | 690/782 [01:18<00:10,  8.80it/s, loss=0.751, v_num=1, val_loss=0.716, val_acc=0.751]\n",
      "Epoch 11:  90%|████████▉ | 700/782 [01:18<00:09,  8.89it/s, loss=0.751, v_num=1, val_loss=0.716, val_acc=0.751]\n",
      "Epoch 11:  91%|█████████ | 710/782 [01:19<00:08,  8.97it/s, loss=0.751, v_num=1, val_loss=0.716, val_acc=0.751]\n",
      "Epoch 11:  92%|█████████▏| 720/782 [01:19<00:06,  9.06it/s, loss=0.751, v_num=1, val_loss=0.716, val_acc=0.751]\n",
      "Epoch 11:  93%|█████████▎| 730/782 [01:19<00:05,  9.14it/s, loss=0.751, v_num=1, val_loss=0.716, val_acc=0.751]\n",
      "Epoch 11:  95%|█████████▍| 740/782 [01:20<00:04,  9.23it/s, loss=0.751, v_num=1, val_loss=0.716, val_acc=0.751]\n",
      "Epoch 11:  96%|█████████▌| 750/782 [01:20<00:03,  9.32it/s, loss=0.751, v_num=1, val_loss=0.716, val_acc=0.751]\n",
      "Epoch 11:  97%|█████████▋| 760/782 [01:20<00:02,  9.40it/s, loss=0.751, v_num=1, val_loss=0.716, val_acc=0.751]\n",
      "Epoch 11:  98%|█████████▊| 770/782 [01:21<00:01,  9.49it/s, loss=0.751, v_num=1, val_loss=0.716, val_acc=0.751]\n",
      "Epoch 11: 100%|█████████▉| 780/782 [01:21<00:00,  9.57it/s, loss=0.751, v_num=1, val_loss=0.716, val_acc=0.751]\n",
      "Epoch 11: 100%|██████████| 782/782 [01:22<00:00,  9.47it/s, loss=0.793, v_num=1, val_loss=0.770, val_acc=0.730]\n",
      "Epoch 12:  81%|████████  | 630/782 [01:19<00:19,  7.92it/s, loss=0.7, v_num=1, val_loss=0.770, val_acc=0.730]   \n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 12:  82%|████████▏ | 640/782 [01:20<00:17,  7.99it/s, loss=0.7, v_num=1, val_loss=0.770, val_acc=0.730]\n",
      "Epoch 12:  83%|████████▎ | 650/782 [01:20<00:16,  8.07it/s, loss=0.7, v_num=1, val_loss=0.770, val_acc=0.730]\n",
      "Epoch 12:  84%|████████▍ | 660/782 [01:21<00:14,  8.15it/s, loss=0.7, v_num=1, val_loss=0.770, val_acc=0.730]\n",
      "Epoch 12:  86%|████████▌ | 670/782 [01:21<00:13,  8.22it/s, loss=0.7, v_num=1, val_loss=0.770, val_acc=0.730]\n",
      "Epoch 12:  87%|████████▋ | 680/782 [01:22<00:12,  8.30it/s, loss=0.7, v_num=1, val_loss=0.770, val_acc=0.730]\n",
      "Epoch 12:  88%|████████▊ | 690/782 [01:22<00:10,  8.39it/s, loss=0.7, v_num=1, val_loss=0.770, val_acc=0.730]\n",
      "Epoch 12:  90%|████████▉ | 700/782 [01:22<00:09,  8.47it/s, loss=0.7, v_num=1, val_loss=0.770, val_acc=0.730]\n",
      "Epoch 12:  91%|█████████ | 710/782 [01:23<00:08,  8.55it/s, loss=0.7, v_num=1, val_loss=0.770, val_acc=0.730]\n",
      "Epoch 12:  92%|█████████▏| 720/782 [01:23<00:07,  8.63it/s, loss=0.7, v_num=1, val_loss=0.770, val_acc=0.730]\n",
      "Epoch 12:  93%|█████████▎| 730/782 [01:23<00:05,  8.71it/s, loss=0.7, v_num=1, val_loss=0.770, val_acc=0.730]\n",
      "Epoch 12:  95%|█████████▍| 740/782 [01:24<00:04,  8.79it/s, loss=0.7, v_num=1, val_loss=0.770, val_acc=0.730]\n",
      "Epoch 12:  96%|█████████▌| 750/782 [01:24<00:03,  8.87it/s, loss=0.7, v_num=1, val_loss=0.770, val_acc=0.730]\n",
      "Epoch 12:  97%|█████████▋| 760/782 [01:24<00:02,  8.96it/s, loss=0.7, v_num=1, val_loss=0.770, val_acc=0.730]\n",
      "Epoch 12:  98%|█████████▊| 770/782 [01:25<00:01,  9.03it/s, loss=0.7, v_num=1, val_loss=0.770, val_acc=0.730]\n",
      "Epoch 12: 100%|█████████▉| 780/782 [01:25<00:00,  9.11it/s, loss=0.7, v_num=1, val_loss=0.770, val_acc=0.730]\n",
      "Epoch 12: 100%|██████████| 782/782 [01:25<00:00,  9.11it/s, loss=0.683, v_num=1, val_loss=0.631, val_acc=0.786]\n",
      "Epoch 13:  81%|████████  | 630/782 [01:21<00:19,  7.75it/s, loss=0.652, v_num=1, val_loss=0.631, val_acc=0.786]\n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 13:  82%|████████▏ | 640/782 [01:21<00:18,  7.84it/s, loss=0.652, v_num=1, val_loss=0.631, val_acc=0.786]\n",
      "Epoch 13:  83%|████████▎ | 650/782 [01:22<00:16,  7.94it/s, loss=0.652, v_num=1, val_loss=0.631, val_acc=0.786]\n",
      "Epoch 13:  84%|████████▍ | 660/782 [01:22<00:15,  8.02it/s, loss=0.652, v_num=1, val_loss=0.631, val_acc=0.786]\n",
      "Epoch 13:  86%|████████▌ | 670/782 [01:22<00:13,  8.11it/s, loss=0.652, v_num=1, val_loss=0.631, val_acc=0.786]\n",
      "Epoch 13:  87%|████████▋ | 680/782 [01:23<00:12,  8.20it/s, loss=0.652, v_num=1, val_loss=0.631, val_acc=0.786]\n",
      "Epoch 13:  88%|████████▊ | 690/782 [01:23<00:11,  8.28it/s, loss=0.652, v_num=1, val_loss=0.631, val_acc=0.786]\n",
      "Epoch 13:  90%|████████▉ | 700/782 [01:23<00:09,  8.37it/s, loss=0.652, v_num=1, val_loss=0.631, val_acc=0.786]\n",
      "Epoch 13:  91%|█████████ | 710/782 [01:24<00:08,  8.45it/s, loss=0.652, v_num=1, val_loss=0.631, val_acc=0.786]\n",
      "Epoch 13:  92%|█████████▏| 720/782 [01:24<00:07,  8.54it/s, loss=0.652, v_num=1, val_loss=0.631, val_acc=0.786]\n",
      "Epoch 13:  93%|█████████▎| 730/782 [01:24<00:06,  8.63it/s, loss=0.652, v_num=1, val_loss=0.631, val_acc=0.786]\n",
      "Epoch 13:  95%|█████████▍| 740/782 [01:25<00:04,  8.71it/s, loss=0.652, v_num=1, val_loss=0.631, val_acc=0.786]\n",
      "Epoch 13:  96%|█████████▌| 750/782 [01:25<00:03,  8.79it/s, loss=0.652, v_num=1, val_loss=0.631, val_acc=0.786]\n",
      "Epoch 13:  97%|█████████▋| 760/782 [01:25<00:02,  8.88it/s, loss=0.652, v_num=1, val_loss=0.631, val_acc=0.786]\n",
      "Epoch 13:  98%|█████████▊| 770/782 [01:26<00:01,  8.96it/s, loss=0.652, v_num=1, val_loss=0.631, val_acc=0.786]\n",
      "Epoch 13: 100%|█████████▉| 780/782 [01:26<00:00,  9.04it/s, loss=0.652, v_num=1, val_loss=0.631, val_acc=0.786]\n",
      "Epoch 13: 100%|██████████| 782/782 [01:26<00:00,  9.03it/s, loss=0.596, v_num=1, val_loss=0.832, val_acc=0.720]\n",
      "Epoch 14:  81%|████████  | 630/782 [01:16<00:18,  8.30it/s, loss=0.642, v_num=1, val_loss=0.832, val_acc=0.720]\n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 14:  82%|████████▏ | 640/782 [01:16<00:16,  8.40it/s, loss=0.642, v_num=1, val_loss=0.832, val_acc=0.720]\n",
      "Epoch 14:  83%|████████▎ | 650/782 [01:16<00:15,  8.49it/s, loss=0.642, v_num=1, val_loss=0.832, val_acc=0.720]\n",
      "Epoch 14:  84%|████████▍ | 660/782 [01:16<00:14,  8.59it/s, loss=0.642, v_num=1, val_loss=0.832, val_acc=0.720]\n",
      "Epoch 14:  86%|████████▌ | 670/782 [01:17<00:12,  8.68it/s, loss=0.642, v_num=1, val_loss=0.832, val_acc=0.720]\n",
      "Epoch 14:  87%|████████▋ | 680/782 [01:17<00:11,  8.78it/s, loss=0.642, v_num=1, val_loss=0.832, val_acc=0.720]\n",
      "Epoch 14:  88%|████████▊ | 690/782 [01:17<00:10,  8.87it/s, loss=0.642, v_num=1, val_loss=0.832, val_acc=0.720]\n",
      "Epoch 14:  90%|████████▉ | 700/782 [01:18<00:09,  8.96it/s, loss=0.642, v_num=1, val_loss=0.832, val_acc=0.720]\n",
      "Epoch 14:  91%|█████████ | 710/782 [01:18<00:07,  9.05it/s, loss=0.642, v_num=1, val_loss=0.832, val_acc=0.720]\n",
      "Epoch 14:  92%|█████████▏| 720/782 [01:18<00:06,  9.15it/s, loss=0.642, v_num=1, val_loss=0.832, val_acc=0.720]\n",
      "Epoch 14:  93%|█████████▎| 730/782 [01:19<00:05,  9.24it/s, loss=0.642, v_num=1, val_loss=0.832, val_acc=0.720]\n",
      "Epoch 14:  95%|█████████▍| 740/782 [01:19<00:04,  9.32it/s, loss=0.642, v_num=1, val_loss=0.832, val_acc=0.720]\n",
      "Epoch 14:  96%|█████████▌| 750/782 [01:19<00:03,  9.41it/s, loss=0.642, v_num=1, val_loss=0.832, val_acc=0.720]\n",
      "Epoch 14:  97%|█████████▋| 760/782 [01:20<00:02,  9.50it/s, loss=0.642, v_num=1, val_loss=0.832, val_acc=0.720]\n",
      "Epoch 14:  98%|█████████▊| 770/782 [01:20<00:01,  9.59it/s, loss=0.642, v_num=1, val_loss=0.832, val_acc=0.720]\n",
      "Epoch 14: 100%|█████████▉| 780/782 [01:20<00:00,  9.68it/s, loss=0.642, v_num=1, val_loss=0.832, val_acc=0.720]\n",
      "Epoch 14: 100%|██████████| 782/782 [01:20<00:00,  9.67it/s, loss=0.601, v_num=1, val_loss=0.839, val_acc=0.733]\n",
      "Epoch 15:  81%|████████  | 630/782 [01:17<00:18,  8.18it/s, loss=0.661, v_num=1, val_loss=0.839, val_acc=0.733] \n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 15:  82%|████████▏ | 640/782 [01:17<00:17,  8.27it/s, loss=0.661, v_num=1, val_loss=0.839, val_acc=0.733]\n",
      "Epoch 15:  83%|████████▎ | 650/782 [01:17<00:15,  8.37it/s, loss=0.661, v_num=1, val_loss=0.839, val_acc=0.733]\n",
      "Epoch 15:  84%|████████▍ | 660/782 [01:18<00:14,  8.46it/s, loss=0.661, v_num=1, val_loss=0.839, val_acc=0.733]\n",
      "Epoch 15:  86%|████████▌ | 670/782 [01:18<00:13,  8.54it/s, loss=0.661, v_num=1, val_loss=0.839, val_acc=0.733]\n",
      "Epoch 15:  87%|████████▋ | 680/782 [01:19<00:11,  8.62it/s, loss=0.661, v_num=1, val_loss=0.839, val_acc=0.733]\n",
      "Epoch 15:  88%|████████▊ | 690/782 [01:19<00:10,  8.70it/s, loss=0.661, v_num=1, val_loss=0.839, val_acc=0.733]\n",
      "Epoch 15:  90%|████████▉ | 700/782 [01:19<00:09,  8.79it/s, loss=0.661, v_num=1, val_loss=0.839, val_acc=0.733]\n",
      "Epoch 15:  91%|█████████ | 710/782 [01:20<00:08,  8.88it/s, loss=0.661, v_num=1, val_loss=0.839, val_acc=0.733]\n",
      "Epoch 15:  92%|█████████▏| 720/782 [01:20<00:06,  8.96it/s, loss=0.661, v_num=1, val_loss=0.839, val_acc=0.733]\n",
      "Epoch 15:  93%|█████████▎| 730/782 [01:20<00:05,  9.05it/s, loss=0.661, v_num=1, val_loss=0.839, val_acc=0.733]\n",
      "Epoch 15:  95%|█████████▍| 740/782 [01:21<00:04,  9.14it/s, loss=0.661, v_num=1, val_loss=0.839, val_acc=0.733]\n",
      "Epoch 15:  96%|█████████▌| 750/782 [01:21<00:03,  9.22it/s, loss=0.661, v_num=1, val_loss=0.839, val_acc=0.733]\n",
      "Epoch 15:  97%|█████████▋| 760/782 [01:21<00:02,  9.31it/s, loss=0.661, v_num=1, val_loss=0.839, val_acc=0.733]\n",
      "Epoch 15:  98%|█████████▊| 770/782 [01:22<00:01,  9.39it/s, loss=0.661, v_num=1, val_loss=0.839, val_acc=0.733]\n",
      "Epoch 15: 100%|█████████▉| 780/782 [01:22<00:00,  9.48it/s, loss=0.661, v_num=1, val_loss=0.839, val_acc=0.733]\n",
      "Epoch 15: 100%|██████████| 782/782 [01:22<00:00,  9.46it/s, loss=0.624, v_num=1, val_loss=0.813, val_acc=0.736]\n",
      "Epoch 16:  81%|████████  | 630/782 [01:11<00:17,  8.77it/s, loss=0.56, v_num=1, val_loss=0.813, val_acc=0.736]  \n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 16:  82%|████████▏ | 640/782 [01:12<00:16,  8.87it/s, loss=0.56, v_num=1, val_loss=0.813, val_acc=0.736]\n",
      "Epoch 16:  83%|████████▎ | 650/782 [01:12<00:14,  8.97it/s, loss=0.56, v_num=1, val_loss=0.813, val_acc=0.736]\n",
      "Epoch 16:  84%|████████▍ | 660/782 [01:12<00:13,  9.07it/s, loss=0.56, v_num=1, val_loss=0.813, val_acc=0.736]\n",
      "Epoch 16:  86%|████████▌ | 670/782 [01:13<00:12,  9.17it/s, loss=0.56, v_num=1, val_loss=0.813, val_acc=0.736]\n",
      "Epoch 16:  87%|████████▋ | 680/782 [01:13<00:11,  9.26it/s, loss=0.56, v_num=1, val_loss=0.813, val_acc=0.736]\n",
      "Epoch 16:  88%|████████▊ | 690/782 [01:13<00:09,  9.36it/s, loss=0.56, v_num=1, val_loss=0.813, val_acc=0.736]\n",
      "Epoch 16:  90%|████████▉ | 700/782 [01:14<00:08,  9.46it/s, loss=0.56, v_num=1, val_loss=0.813, val_acc=0.736]\n",
      "Epoch 16:  91%|█████████ | 710/782 [01:14<00:07,  9.55it/s, loss=0.56, v_num=1, val_loss=0.813, val_acc=0.736]\n",
      "Epoch 16:  92%|█████████▏| 720/782 [01:14<00:06,  9.64it/s, loss=0.56, v_num=1, val_loss=0.813, val_acc=0.736]\n",
      "Epoch 16:  93%|█████████▎| 730/782 [01:15<00:05,  9.74it/s, loss=0.56, v_num=1, val_loss=0.813, val_acc=0.736]\n",
      "Epoch 16:  95%|█████████▍| 740/782 [01:15<00:04,  9.83it/s, loss=0.56, v_num=1, val_loss=0.813, val_acc=0.736]\n",
      "Epoch 16:  96%|█████████▌| 750/782 [01:15<00:03,  9.92it/s, loss=0.56, v_num=1, val_loss=0.813, val_acc=0.736]\n",
      "Epoch 16:  97%|█████████▋| 760/782 [01:16<00:02, 10.01it/s, loss=0.56, v_num=1, val_loss=0.813, val_acc=0.736]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 16:  98%|█████████▊| 770/782 [01:16<00:01, 10.10it/s, loss=0.56, v_num=1, val_loss=0.813, val_acc=0.736]\n",
      "Epoch 16: 100%|█████████▉| 780/782 [01:16<00:00, 10.19it/s, loss=0.56, v_num=1, val_loss=0.813, val_acc=0.736]\n",
      "Epoch 16: 100%|██████████| 782/782 [01:16<00:00, 10.18it/s, loss=0.571, v_num=1, val_loss=0.781, val_acc=0.742]\n",
      "Epoch 17:  81%|████████  | 630/782 [01:17<00:18,  8.17it/s, loss=0.69, v_num=1, val_loss=0.781, val_acc=0.742]  \n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 17:  82%|████████▏ | 640/782 [01:17<00:17,  8.26it/s, loss=0.69, v_num=1, val_loss=0.781, val_acc=0.742]\n",
      "Epoch 17:  83%|████████▎ | 650/782 [01:17<00:15,  8.35it/s, loss=0.69, v_num=1, val_loss=0.781, val_acc=0.742]\n",
      "Epoch 17:  84%|████████▍ | 660/782 [01:18<00:14,  8.44it/s, loss=0.69, v_num=1, val_loss=0.781, val_acc=0.742]\n",
      "Epoch 17:  86%|████████▌ | 670/782 [01:18<00:13,  8.54it/s, loss=0.69, v_num=1, val_loss=0.781, val_acc=0.742]\n",
      "Epoch 17:  87%|████████▋ | 680/782 [01:18<00:11,  8.63it/s, loss=0.69, v_num=1, val_loss=0.781, val_acc=0.742]\n",
      "Epoch 17:  88%|████████▊ | 690/782 [01:19<00:10,  8.72it/s, loss=0.69, v_num=1, val_loss=0.781, val_acc=0.742]\n",
      "Epoch 17:  90%|████████▉ | 700/782 [01:19<00:09,  8.82it/s, loss=0.69, v_num=1, val_loss=0.781, val_acc=0.742]\n",
      "Epoch 17:  91%|█████████ | 710/782 [01:19<00:08,  8.91it/s, loss=0.69, v_num=1, val_loss=0.781, val_acc=0.742]\n",
      "Epoch 17:  92%|█████████▏| 720/782 [01:20<00:06,  9.00it/s, loss=0.69, v_num=1, val_loss=0.781, val_acc=0.742]\n",
      "Epoch 17:  93%|█████████▎| 730/782 [01:20<00:05,  9.08it/s, loss=0.69, v_num=1, val_loss=0.781, val_acc=0.742]\n",
      "Epoch 17:  95%|█████████▍| 740/782 [01:20<00:04,  9.17it/s, loss=0.69, v_num=1, val_loss=0.781, val_acc=0.742]\n",
      "Epoch 17:  96%|█████████▌| 750/782 [01:21<00:03,  9.26it/s, loss=0.69, v_num=1, val_loss=0.781, val_acc=0.742]\n",
      "Epoch 17:  97%|█████████▋| 760/782 [01:21<00:02,  9.35it/s, loss=0.69, v_num=1, val_loss=0.781, val_acc=0.742]\n",
      "Epoch 17:  98%|█████████▊| 770/782 [01:21<00:01,  9.44it/s, loss=0.69, v_num=1, val_loss=0.781, val_acc=0.742]\n",
      "Epoch 17: 100%|█████████▉| 780/782 [01:22<00:00,  9.52it/s, loss=0.69, v_num=1, val_loss=0.781, val_acc=0.742]\n",
      "Epoch 17: 100%|██████████| 782/782 [01:22<00:00,  9.52it/s, loss=0.684, v_num=1, val_loss=0.585, val_acc=0.798]\n",
      "Epoch 18:  81%|████████  | 630/782 [01:12<00:17,  8.67it/s, loss=0.63, v_num=1, val_loss=0.585, val_acc=0.798]  \n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 18:  82%|████████▏ | 640/782 [01:13<00:16,  8.77it/s, loss=0.63, v_num=1, val_loss=0.585, val_acc=0.798]\n",
      "Epoch 18:  83%|████████▎ | 650/782 [01:13<00:14,  8.87it/s, loss=0.63, v_num=1, val_loss=0.585, val_acc=0.798]\n",
      "Epoch 18:  84%|████████▍ | 660/782 [01:13<00:13,  8.96it/s, loss=0.63, v_num=1, val_loss=0.585, val_acc=0.798]\n",
      "Epoch 18:  86%|████████▌ | 670/782 [01:14<00:12,  9.06it/s, loss=0.63, v_num=1, val_loss=0.585, val_acc=0.798]\n",
      "Epoch 18:  87%|████████▋ | 680/782 [01:14<00:11,  9.16it/s, loss=0.63, v_num=1, val_loss=0.585, val_acc=0.798]\n",
      "Epoch 18:  88%|████████▊ | 690/782 [01:14<00:09,  9.25it/s, loss=0.63, v_num=1, val_loss=0.585, val_acc=0.798]\n",
      "Epoch 18:  90%|████████▉ | 700/782 [01:14<00:08,  9.35it/s, loss=0.63, v_num=1, val_loss=0.585, val_acc=0.798]\n",
      "Epoch 18:  91%|█████████ | 710/782 [01:15<00:07,  9.44it/s, loss=0.63, v_num=1, val_loss=0.585, val_acc=0.798]\n",
      "Epoch 18:  92%|█████████▏| 720/782 [01:15<00:06,  9.54it/s, loss=0.63, v_num=1, val_loss=0.585, val_acc=0.798]\n",
      "Epoch 18:  93%|█████████▎| 730/782 [01:15<00:05,  9.63it/s, loss=0.63, v_num=1, val_loss=0.585, val_acc=0.798]\n",
      "Epoch 18:  95%|█████████▍| 740/782 [01:16<00:04,  9.72it/s, loss=0.63, v_num=1, val_loss=0.585, val_acc=0.798]\n",
      "Epoch 18:  96%|█████████▌| 750/782 [01:16<00:03,  9.81it/s, loss=0.63, v_num=1, val_loss=0.585, val_acc=0.798]\n",
      "Epoch 18:  97%|█████████▋| 760/782 [01:16<00:02,  9.90it/s, loss=0.63, v_num=1, val_loss=0.585, val_acc=0.798]\n",
      "Epoch 18:  98%|█████████▊| 770/782 [01:17<00:01,  9.99it/s, loss=0.63, v_num=1, val_loss=0.585, val_acc=0.798]\n",
      "Epoch 18: 100%|█████████▉| 780/782 [01:17<00:00, 10.08it/s, loss=0.63, v_num=1, val_loss=0.585, val_acc=0.798]\n",
      "Epoch 18: 100%|██████████| 782/782 [01:17<00:00, 10.07it/s, loss=0.59, v_num=1, val_loss=0.629, val_acc=0.790]\n",
      "Epoch 19:  81%|████████  | 630/782 [01:19<00:19,  7.89it/s, loss=0.605, v_num=1, val_loss=0.629, val_acc=0.790]\n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 19:  82%|████████▏ | 640/782 [01:20<00:17,  7.98it/s, loss=0.605, v_num=1, val_loss=0.629, val_acc=0.790]\n",
      "Epoch 19:  83%|████████▎ | 650/782 [01:20<00:16,  8.07it/s, loss=0.605, v_num=1, val_loss=0.629, val_acc=0.790]\n",
      "Epoch 19:  84%|████████▍ | 660/782 [01:20<00:14,  8.17it/s, loss=0.605, v_num=1, val_loss=0.629, val_acc=0.790]\n",
      "Epoch 19:  86%|████████▌ | 670/782 [01:21<00:13,  8.26it/s, loss=0.605, v_num=1, val_loss=0.629, val_acc=0.790]\n",
      "Epoch 19:  87%|████████▋ | 680/782 [01:21<00:12,  8.35it/s, loss=0.605, v_num=1, val_loss=0.629, val_acc=0.790]\n",
      "Epoch 19:  88%|████████▊ | 690/782 [01:21<00:10,  8.44it/s, loss=0.605, v_num=1, val_loss=0.629, val_acc=0.790]\n",
      "Epoch 19:  90%|████████▉ | 700/782 [01:22<00:09,  8.53it/s, loss=0.605, v_num=1, val_loss=0.629, val_acc=0.790]\n",
      "Epoch 19:  91%|█████████ | 710/782 [01:22<00:08,  8.62it/s, loss=0.605, v_num=1, val_loss=0.629, val_acc=0.790]\n",
      "Epoch 19:  92%|█████████▏| 720/782 [01:22<00:07,  8.71it/s, loss=0.605, v_num=1, val_loss=0.629, val_acc=0.790]\n",
      "Epoch 19:  93%|█████████▎| 730/782 [01:23<00:05,  8.79it/s, loss=0.605, v_num=1, val_loss=0.629, val_acc=0.790]\n",
      "Epoch 19:  95%|█████████▍| 740/782 [01:23<00:04,  8.88it/s, loss=0.605, v_num=1, val_loss=0.629, val_acc=0.790]\n",
      "Epoch 19:  96%|█████████▌| 750/782 [01:23<00:03,  8.97it/s, loss=0.605, v_num=1, val_loss=0.629, val_acc=0.790]\n",
      "Epoch 19:  97%|█████████▋| 760/782 [01:24<00:02,  9.05it/s, loss=0.605, v_num=1, val_loss=0.629, val_acc=0.790]\n",
      "Epoch 19:  98%|█████████▊| 770/782 [01:24<00:01,  9.14it/s, loss=0.605, v_num=1, val_loss=0.629, val_acc=0.790]\n",
      "Epoch 19: 100%|█████████▉| 780/782 [01:24<00:00,  9.22it/s, loss=0.605, v_num=1, val_loss=0.629, val_acc=0.790]\n",
      "Epoch 19: 100%|██████████| 782/782 [01:24<00:00,  9.22it/s, loss=0.607, v_num=1, val_loss=0.594, val_acc=0.784]\n",
      "Epoch 20:  81%|████████  | 630/782 [01:15<00:18,  8.34it/s, loss=0.638, v_num=1, val_loss=0.594, val_acc=0.784] \n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 20:  82%|████████▏ | 640/782 [01:16<00:16,  8.43it/s, loss=0.638, v_num=1, val_loss=0.594, val_acc=0.784]\n",
      "Epoch 20:  83%|████████▎ | 650/782 [01:16<00:15,  8.52it/s, loss=0.638, v_num=1, val_loss=0.594, val_acc=0.784]\n",
      "Epoch 20:  84%|████████▍ | 660/782 [01:16<00:14,  8.60it/s, loss=0.638, v_num=1, val_loss=0.594, val_acc=0.784]\n",
      "Epoch 20:  86%|████████▌ | 670/782 [01:17<00:12,  8.68it/s, loss=0.638, v_num=1, val_loss=0.594, val_acc=0.784]\n",
      "Epoch 20:  87%|████████▋ | 680/782 [01:17<00:11,  8.76it/s, loss=0.638, v_num=1, val_loss=0.594, val_acc=0.784]\n",
      "Epoch 20:  88%|████████▊ | 690/782 [01:18<00:10,  8.85it/s, loss=0.638, v_num=1, val_loss=0.594, val_acc=0.784]\n",
      "Epoch 20:  90%|████████▉ | 700/782 [01:18<00:09,  8.94it/s, loss=0.638, v_num=1, val_loss=0.594, val_acc=0.784]\n",
      "Epoch 20:  91%|█████████ | 710/782 [01:18<00:07,  9.03it/s, loss=0.638, v_num=1, val_loss=0.594, val_acc=0.784]\n",
      "Epoch 20:  92%|█████████▏| 720/782 [01:19<00:06,  9.10it/s, loss=0.638, v_num=1, val_loss=0.594, val_acc=0.784]\n",
      "Epoch 20:  93%|█████████▎| 730/782 [01:19<00:05,  9.19it/s, loss=0.638, v_num=1, val_loss=0.594, val_acc=0.784]\n",
      "Epoch 20:  95%|█████████▍| 740/782 [01:19<00:04,  9.27it/s, loss=0.638, v_num=1, val_loss=0.594, val_acc=0.784]\n",
      "Epoch 20:  96%|█████████▌| 750/782 [01:20<00:03,  9.36it/s, loss=0.638, v_num=1, val_loss=0.594, val_acc=0.784]\n",
      "Epoch 20:  97%|█████████▋| 760/782 [01:20<00:02,  9.45it/s, loss=0.638, v_num=1, val_loss=0.594, val_acc=0.784]\n",
      "Epoch 20:  98%|█████████▊| 770/782 [01:21<00:01,  9.52it/s, loss=0.638, v_num=1, val_loss=0.594, val_acc=0.784]\n",
      "Epoch 20: 100%|█████████▉| 780/782 [01:21<00:00,  9.60it/s, loss=0.638, v_num=1, val_loss=0.594, val_acc=0.784]\n",
      "Epoch 20: 100%|██████████| 782/782 [01:21<00:00,  9.60it/s, loss=0.595, v_num=1, val_loss=0.662, val_acc=0.776]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 21:  81%|████████  | 630/782 [01:20<00:19,  7.88it/s, loss=0.582, v_num=1, val_loss=0.662, val_acc=0.776]\n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 21:  82%|████████▏ | 640/782 [01:20<00:17,  7.97it/s, loss=0.582, v_num=1, val_loss=0.662, val_acc=0.776]\n",
      "Epoch 21:  83%|████████▎ | 650/782 [01:20<00:16,  8.06it/s, loss=0.582, v_num=1, val_loss=0.662, val_acc=0.776]\n",
      "Epoch 21:  84%|████████▍ | 660/782 [01:21<00:14,  8.15it/s, loss=0.582, v_num=1, val_loss=0.662, val_acc=0.776]\n",
      "Epoch 21:  86%|████████▌ | 670/782 [01:21<00:13,  8.24it/s, loss=0.582, v_num=1, val_loss=0.662, val_acc=0.776]\n",
      "Epoch 21:  87%|████████▋ | 680/782 [01:21<00:12,  8.32it/s, loss=0.582, v_num=1, val_loss=0.662, val_acc=0.776]\n",
      "Epoch 21:  88%|████████▊ | 690/782 [01:22<00:10,  8.41it/s, loss=0.582, v_num=1, val_loss=0.662, val_acc=0.776]\n",
      "Epoch 21:  90%|████████▉ | 700/782 [01:22<00:09,  8.50it/s, loss=0.582, v_num=1, val_loss=0.662, val_acc=0.776]\n",
      "Epoch 21:  91%|█████████ | 710/782 [01:22<00:08,  8.58it/s, loss=0.582, v_num=1, val_loss=0.662, val_acc=0.776]\n",
      "Epoch 21:  92%|█████████▏| 720/782 [01:23<00:07,  8.66it/s, loss=0.582, v_num=1, val_loss=0.662, val_acc=0.776]\n",
      "Epoch 21:  93%|█████████▎| 730/782 [01:23<00:05,  8.75it/s, loss=0.582, v_num=1, val_loss=0.662, val_acc=0.776]\n",
      "Epoch 21:  95%|█████████▍| 740/782 [01:23<00:04,  8.83it/s, loss=0.582, v_num=1, val_loss=0.662, val_acc=0.776]\n",
      "Epoch 21:  96%|█████████▌| 750/782 [01:24<00:03,  8.91it/s, loss=0.582, v_num=1, val_loss=0.662, val_acc=0.776]\n",
      "Epoch 21:  97%|█████████▋| 760/782 [01:24<00:02,  8.99it/s, loss=0.582, v_num=1, val_loss=0.662, val_acc=0.776]\n",
      "Epoch 21:  98%|█████████▊| 770/782 [01:24<00:01,  9.07it/s, loss=0.582, v_num=1, val_loss=0.662, val_acc=0.776]\n",
      "Epoch 21: 100%|█████████▉| 780/782 [01:25<00:00,  9.15it/s, loss=0.582, v_num=1, val_loss=0.662, val_acc=0.776]\n",
      "Epoch 21: 100%|██████████| 782/782 [01:26<00:00,  9.08it/s, loss=0.567, v_num=1, val_loss=0.618, val_acc=0.789]\n",
      "Epoch 22:  81%|████████  | 630/782 [01:42<00:24,  6.18it/s, loss=0.559, v_num=1, val_loss=0.618, val_acc=0.789] \n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 22:  82%|████████▏ | 640/782 [01:42<00:22,  6.25it/s, loss=0.559, v_num=1, val_loss=0.618, val_acc=0.789]\n",
      "Epoch 22:  83%|████████▎ | 650/782 [01:42<00:20,  6.32it/s, loss=0.559, v_num=1, val_loss=0.618, val_acc=0.789]\n",
      "Epoch 22:  84%|████████▍ | 660/782 [01:43<00:19,  6.40it/s, loss=0.559, v_num=1, val_loss=0.618, val_acc=0.789]\n",
      "Epoch 22:  86%|████████▌ | 670/782 [01:43<00:17,  6.47it/s, loss=0.559, v_num=1, val_loss=0.618, val_acc=0.789]\n",
      "Epoch 22:  87%|████████▋ | 680/782 [01:44<00:15,  6.54it/s, loss=0.559, v_num=1, val_loss=0.618, val_acc=0.789]\n",
      "Epoch 22:  88%|████████▊ | 690/782 [01:44<00:13,  6.61it/s, loss=0.559, v_num=1, val_loss=0.618, val_acc=0.789]\n",
      "Epoch 22:  90%|████████▉ | 700/782 [01:44<00:12,  6.68it/s, loss=0.559, v_num=1, val_loss=0.618, val_acc=0.789]\n",
      "Epoch 22:  91%|█████████ | 710/782 [01:45<00:10,  6.75it/s, loss=0.559, v_num=1, val_loss=0.618, val_acc=0.789]\n",
      "Epoch 22:  92%|█████████▏| 720/782 [01:45<00:09,  6.82it/s, loss=0.559, v_num=1, val_loss=0.618, val_acc=0.789]\n",
      "Epoch 22:  93%|█████████▎| 730/782 [01:46<00:07,  6.90it/s, loss=0.559, v_num=1, val_loss=0.618, val_acc=0.789]\n",
      "Epoch 22:  95%|█████████▍| 740/782 [01:46<00:06,  6.97it/s, loss=0.559, v_num=1, val_loss=0.618, val_acc=0.789]\n",
      "Epoch 22:  96%|█████████▌| 750/782 [01:46<00:04,  7.04it/s, loss=0.559, v_num=1, val_loss=0.618, val_acc=0.789]\n",
      "Epoch 22:  97%|█████████▋| 760/782 [01:47<00:03,  7.11it/s, loss=0.559, v_num=1, val_loss=0.618, val_acc=0.789]\n",
      "Epoch 22:  98%|█████████▊| 770/782 [01:47<00:01,  7.18it/s, loss=0.559, v_num=1, val_loss=0.618, val_acc=0.789]\n",
      "Epoch 22: 100%|█████████▉| 780/782 [01:47<00:00,  7.25it/s, loss=0.559, v_num=1, val_loss=0.618, val_acc=0.789]\n",
      "Epoch 22: 100%|██████████| 782/782 [01:48<00:00,  7.24it/s, loss=0.55, v_num=1, val_loss=0.590, val_acc=0.805] \n",
      "Epoch 23:  81%|████████  | 630/782 [01:22<00:19,  7.66it/s, loss=0.464, v_num=1, val_loss=0.590, val_acc=0.805]\n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 23:  82%|████████▏ | 640/782 [01:22<00:18,  7.75it/s, loss=0.464, v_num=1, val_loss=0.590, val_acc=0.805]\n",
      "Epoch 23:  83%|████████▎ | 650/782 [01:23<00:16,  7.84it/s, loss=0.464, v_num=1, val_loss=0.590, val_acc=0.805]\n",
      "Epoch 23:  84%|████████▍ | 660/782 [01:23<00:15,  7.92it/s, loss=0.464, v_num=1, val_loss=0.590, val_acc=0.805]\n",
      "Epoch 23:  86%|████████▌ | 670/782 [01:23<00:13,  8.01it/s, loss=0.464, v_num=1, val_loss=0.590, val_acc=0.805]\n",
      "Epoch 23:  87%|████████▋ | 680/782 [01:24<00:12,  8.10it/s, loss=0.464, v_num=1, val_loss=0.590, val_acc=0.805]\n",
      "Epoch 23:  88%|████████▊ | 690/782 [01:24<00:11,  8.19it/s, loss=0.464, v_num=1, val_loss=0.590, val_acc=0.805]\n",
      "Epoch 23:  90%|████████▉ | 700/782 [01:24<00:09,  8.28it/s, loss=0.464, v_num=1, val_loss=0.590, val_acc=0.805]\n",
      "Epoch 23:  91%|█████████ | 710/782 [01:25<00:08,  8.36it/s, loss=0.464, v_num=1, val_loss=0.590, val_acc=0.805]\n",
      "Epoch 23:  92%|█████████▏| 720/782 [01:25<00:07,  8.45it/s, loss=0.464, v_num=1, val_loss=0.590, val_acc=0.805]\n",
      "Epoch 23:  93%|█████████▎| 730/782 [01:25<00:06,  8.54it/s, loss=0.464, v_num=1, val_loss=0.590, val_acc=0.805]\n",
      "Epoch 23:  95%|█████████▍| 740/782 [01:25<00:04,  8.62it/s, loss=0.464, v_num=1, val_loss=0.590, val_acc=0.805]\n",
      "Epoch 23:  96%|█████████▌| 750/782 [01:26<00:03,  8.70it/s, loss=0.464, v_num=1, val_loss=0.590, val_acc=0.805]\n",
      "Epoch 23:  97%|█████████▋| 760/782 [01:26<00:02,  8.79it/s, loss=0.464, v_num=1, val_loss=0.590, val_acc=0.805]\n",
      "Epoch 23:  98%|█████████▊| 770/782 [01:26<00:01,  8.87it/s, loss=0.464, v_num=1, val_loss=0.590, val_acc=0.805]\n",
      "Epoch 23: 100%|█████████▉| 780/782 [01:27<00:00,  8.95it/s, loss=0.464, v_num=1, val_loss=0.590, val_acc=0.805]\n",
      "Epoch 23: 100%|██████████| 782/782 [01:27<00:00,  8.91it/s, loss=0.481, v_num=1, val_loss=0.629, val_acc=0.802]\n",
      "Epoch 24:  81%|████████  | 630/782 [01:14<00:17,  8.48it/s, loss=0.509, v_num=1, val_loss=0.629, val_acc=0.802] \n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 24:  82%|████████▏ | 640/782 [01:14<00:16,  8.57it/s, loss=0.509, v_num=1, val_loss=0.629, val_acc=0.802]\n",
      "Epoch 24:  83%|████████▎ | 650/782 [01:15<00:15,  8.67it/s, loss=0.509, v_num=1, val_loss=0.629, val_acc=0.802]\n",
      "Epoch 24:  84%|████████▍ | 660/782 [01:15<00:13,  8.76it/s, loss=0.509, v_num=1, val_loss=0.629, val_acc=0.802]\n",
      "Epoch 24:  86%|████████▌ | 670/782 [01:15<00:12,  8.84it/s, loss=0.509, v_num=1, val_loss=0.629, val_acc=0.802]\n",
      "Epoch 24:  87%|████████▋ | 680/782 [01:16<00:11,  8.92it/s, loss=0.509, v_num=1, val_loss=0.629, val_acc=0.802]\n",
      "Epoch 24:  88%|████████▊ | 690/782 [01:16<00:10,  9.01it/s, loss=0.509, v_num=1, val_loss=0.629, val_acc=0.802]\n",
      "Epoch 24:  90%|████████▉ | 700/782 [01:17<00:09,  9.10it/s, loss=0.509, v_num=1, val_loss=0.629, val_acc=0.802]\n",
      "Epoch 24:  91%|█████████ | 710/782 [01:17<00:07,  9.18it/s, loss=0.509, v_num=1, val_loss=0.629, val_acc=0.802]\n",
      "Epoch 24:  92%|█████████▏| 720/782 [01:17<00:06,  9.27it/s, loss=0.509, v_num=1, val_loss=0.629, val_acc=0.802]\n",
      "Epoch 24:  93%|█████████▎| 730/782 [01:18<00:05,  9.36it/s, loss=0.509, v_num=1, val_loss=0.629, val_acc=0.802]\n",
      "Epoch 24:  95%|█████████▍| 740/782 [01:18<00:04,  9.44it/s, loss=0.509, v_num=1, val_loss=0.629, val_acc=0.802]\n",
      "Epoch 24:  96%|█████████▌| 750/782 [01:18<00:03,  9.53it/s, loss=0.509, v_num=1, val_loss=0.629, val_acc=0.802]\n",
      "Epoch 24:  97%|█████████▋| 760/782 [01:19<00:02,  9.62it/s, loss=0.509, v_num=1, val_loss=0.629, val_acc=0.802]\n",
      "Epoch 24:  98%|█████████▊| 770/782 [01:19<00:01,  9.70it/s, loss=0.509, v_num=1, val_loss=0.629, val_acc=0.802]\n",
      "Epoch 24: 100%|█████████▉| 780/782 [01:19<00:00,  9.79it/s, loss=0.509, v_num=1, val_loss=0.629, val_acc=0.802]\n",
      "Epoch 24: 100%|██████████| 782/782 [01:20<00:00,  9.74it/s, loss=0.468, v_num=1, val_loss=0.567, val_acc=0.807]\n",
      "Epoch 25:  81%|████████  | 630/782 [01:33<00:22,  6.72it/s, loss=0.437, v_num=1, val_loss=0.567, val_acc=0.807]\n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 25:  82%|████████▏ | 640/782 [01:34<00:20,  6.80it/s, loss=0.437, v_num=1, val_loss=0.567, val_acc=0.807]\n",
      "Epoch 25:  83%|████████▎ | 650/782 [01:34<00:19,  6.88it/s, loss=0.437, v_num=1, val_loss=0.567, val_acc=0.807]\n",
      "Epoch 25:  84%|████████▍ | 660/782 [01:34<00:17,  6.97it/s, loss=0.437, v_num=1, val_loss=0.567, val_acc=0.807]\n",
      "Epoch 25:  86%|████████▌ | 670/782 [01:35<00:15,  7.05it/s, loss=0.437, v_num=1, val_loss=0.567, val_acc=0.807]\n",
      "Epoch 25:  87%|████████▋ | 680/782 [01:35<00:14,  7.13it/s, loss=0.437, v_num=1, val_loss=0.567, val_acc=0.807]\n",
      "Epoch 25:  88%|████████▊ | 690/782 [01:35<00:12,  7.21it/s, loss=0.437, v_num=1, val_loss=0.567, val_acc=0.807]\n",
      "Epoch 25:  90%|████████▉ | 700/782 [01:36<00:11,  7.29it/s, loss=0.437, v_num=1, val_loss=0.567, val_acc=0.807]\n",
      "Epoch 25:  91%|█████████ | 710/782 [01:36<00:09,  7.37it/s, loss=0.437, v_num=1, val_loss=0.567, val_acc=0.807]\n",
      "Epoch 25:  92%|█████████▏| 720/782 [01:36<00:08,  7.45it/s, loss=0.437, v_num=1, val_loss=0.567, val_acc=0.807]\n",
      "Epoch 25:  93%|█████████▎| 730/782 [01:37<00:06,  7.53it/s, loss=0.437, v_num=1, val_loss=0.567, val_acc=0.807]\n",
      "Epoch 25:  95%|█████████▍| 740/782 [01:37<00:05,  7.61it/s, loss=0.437, v_num=1, val_loss=0.567, val_acc=0.807]\n",
      "Epoch 25:  96%|█████████▌| 750/782 [01:37<00:04,  7.69it/s, loss=0.437, v_num=1, val_loss=0.567, val_acc=0.807]\n",
      "Epoch 25:  97%|█████████▋| 760/782 [01:37<00:02,  7.77it/s, loss=0.437, v_num=1, val_loss=0.567, val_acc=0.807]\n",
      "Epoch 25:  98%|█████████▊| 770/782 [01:38<00:01,  7.84it/s, loss=0.437, v_num=1, val_loss=0.567, val_acc=0.807]\n",
      "Epoch 25: 100%|█████████▉| 780/782 [01:38<00:00,  7.92it/s, loss=0.437, v_num=1, val_loss=0.567, val_acc=0.807]\n",
      "Epoch 25: 100%|██████████| 782/782 [01:38<00:00,  7.92it/s, loss=0.422, v_num=1, val_loss=0.450, val_acc=0.844]\n",
      "Epoch 26:  81%|████████  | 630/782 [01:21<00:19,  7.76it/s, loss=0.37, v_num=1, val_loss=0.450, val_acc=0.844]  \n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 26:  82%|████████▏ | 640/782 [01:21<00:18,  7.84it/s, loss=0.37, v_num=1, val_loss=0.450, val_acc=0.844]\n",
      "Epoch 26:  83%|████████▎ | 650/782 [01:22<00:16,  7.93it/s, loss=0.37, v_num=1, val_loss=0.450, val_acc=0.844]\n",
      "Epoch 26:  84%|████████▍ | 660/782 [01:22<00:15,  8.01it/s, loss=0.37, v_num=1, val_loss=0.450, val_acc=0.844]\n",
      "Epoch 26:  86%|████████▌ | 670/782 [01:22<00:13,  8.10it/s, loss=0.37, v_num=1, val_loss=0.450, val_acc=0.844]\n",
      "Epoch 26:  87%|████████▋ | 680/782 [01:23<00:12,  8.18it/s, loss=0.37, v_num=1, val_loss=0.450, val_acc=0.844]\n",
      "Epoch 26:  88%|████████▊ | 690/782 [01:23<00:11,  8.26it/s, loss=0.37, v_num=1, val_loss=0.450, val_acc=0.844]\n",
      "Epoch 26:  90%|████████▉ | 700/782 [01:23<00:09,  8.35it/s, loss=0.37, v_num=1, val_loss=0.450, val_acc=0.844]\n",
      "Epoch 26:  91%|█████████ | 710/782 [01:24<00:08,  8.43it/s, loss=0.37, v_num=1, val_loss=0.450, val_acc=0.844]\n",
      "Epoch 26:  92%|█████████▏| 720/782 [01:24<00:07,  8.51it/s, loss=0.37, v_num=1, val_loss=0.450, val_acc=0.844]\n",
      "Epoch 26:  93%|█████████▎| 730/782 [01:25<00:06,  8.59it/s, loss=0.37, v_num=1, val_loss=0.450, val_acc=0.844]\n",
      "Epoch 26:  95%|█████████▍| 740/782 [01:25<00:04,  8.67it/s, loss=0.37, v_num=1, val_loss=0.450, val_acc=0.844]\n",
      "Epoch 26:  96%|█████████▌| 750/782 [01:25<00:03,  8.75it/s, loss=0.37, v_num=1, val_loss=0.450, val_acc=0.844]\n",
      "Epoch 26:  97%|█████████▋| 760/782 [01:26<00:02,  8.83it/s, loss=0.37, v_num=1, val_loss=0.450, val_acc=0.844]\n",
      "Epoch 26:  98%|█████████▊| 770/782 [01:26<00:01,  8.91it/s, loss=0.37, v_num=1, val_loss=0.450, val_acc=0.844]\n",
      "Epoch 26: 100%|█████████▉| 780/782 [01:26<00:00,  8.99it/s, loss=0.37, v_num=1, val_loss=0.450, val_acc=0.844]\n",
      "Epoch 26: 100%|██████████| 782/782 [01:27<00:00,  9.00it/s, loss=0.377, v_num=1, val_loss=0.424, val_acc=0.856]\n",
      "Epoch 27:  81%|████████  | 630/782 [01:26<00:20,  7.30it/s, loss=0.402, v_num=1, val_loss=0.424, val_acc=0.856] \n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 27:  82%|████████▏ | 640/782 [01:26<00:19,  7.38it/s, loss=0.402, v_num=1, val_loss=0.424, val_acc=0.856]\n",
      "Epoch 27:  83%|████████▎ | 650/782 [01:27<00:17,  7.47it/s, loss=0.402, v_num=1, val_loss=0.424, val_acc=0.856]\n",
      "Epoch 27:  84%|████████▍ | 660/782 [01:27<00:16,  7.55it/s, loss=0.402, v_num=1, val_loss=0.424, val_acc=0.856]\n",
      "Epoch 27:  86%|████████▌ | 670/782 [01:27<00:14,  7.64it/s, loss=0.402, v_num=1, val_loss=0.424, val_acc=0.856]\n",
      "Epoch 27:  87%|████████▋ | 680/782 [01:28<00:13,  7.72it/s, loss=0.402, v_num=1, val_loss=0.424, val_acc=0.856]\n",
      "Epoch 27:  88%|████████▊ | 690/782 [01:28<00:11,  7.81it/s, loss=0.402, v_num=1, val_loss=0.424, val_acc=0.856]\n",
      "Epoch 27:  90%|████████▉ | 700/782 [01:28<00:10,  7.89it/s, loss=0.402, v_num=1, val_loss=0.424, val_acc=0.856]\n",
      "Epoch 27:  91%|█████████ | 710/782 [01:29<00:09,  7.97it/s, loss=0.402, v_num=1, val_loss=0.424, val_acc=0.856]\n",
      "Epoch 27:  92%|█████████▏| 720/782 [01:29<00:07,  8.06it/s, loss=0.402, v_num=1, val_loss=0.424, val_acc=0.856]\n",
      "Epoch 27:  93%|█████████▎| 730/782 [01:29<00:06,  8.14it/s, loss=0.402, v_num=1, val_loss=0.424, val_acc=0.856]\n",
      "Epoch 27:  95%|█████████▍| 740/782 [01:30<00:05,  8.22it/s, loss=0.402, v_num=1, val_loss=0.424, val_acc=0.856]\n",
      "Epoch 27:  96%|█████████▌| 750/782 [01:30<00:03,  8.30it/s, loss=0.402, v_num=1, val_loss=0.424, val_acc=0.856]\n",
      "Epoch 27:  97%|█████████▋| 760/782 [01:30<00:02,  8.38it/s, loss=0.402, v_num=1, val_loss=0.424, val_acc=0.856]\n",
      "Epoch 27:  98%|█████████▊| 770/782 [01:31<00:01,  8.46it/s, loss=0.402, v_num=1, val_loss=0.424, val_acc=0.856]\n",
      "Epoch 27: 100%|█████████▉| 780/782 [01:31<00:00,  8.54it/s, loss=0.402, v_num=1, val_loss=0.424, val_acc=0.856]\n",
      "Epoch 27: 100%|██████████| 782/782 [01:31<00:00,  8.52it/s, loss=0.407, v_num=1, val_loss=0.434, val_acc=0.855]\n",
      "Epoch 28:  81%|████████  | 630/782 [01:16<00:18,  8.23it/s, loss=0.284, v_num=1, val_loss=0.434, val_acc=0.855]\n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 28:  82%|████████▏ | 640/782 [01:17<00:17,  8.32it/s, loss=0.284, v_num=1, val_loss=0.434, val_acc=0.855]\n",
      "Epoch 28:  83%|████████▎ | 650/782 [01:17<00:15,  8.42it/s, loss=0.284, v_num=1, val_loss=0.434, val_acc=0.855]\n",
      "Epoch 28:  84%|████████▍ | 660/782 [01:17<00:14,  8.50it/s, loss=0.284, v_num=1, val_loss=0.434, val_acc=0.855]\n",
      "Epoch 28:  86%|████████▌ | 670/782 [01:18<00:13,  8.60it/s, loss=0.284, v_num=1, val_loss=0.434, val_acc=0.855]\n",
      "Epoch 28:  87%|████████▋ | 680/782 [01:18<00:11,  8.69it/s, loss=0.284, v_num=1, val_loss=0.434, val_acc=0.855]\n",
      "Epoch 28:  88%|████████▊ | 690/782 [01:18<00:10,  8.78it/s, loss=0.284, v_num=1, val_loss=0.434, val_acc=0.855]\n",
      "Epoch 28:  90%|████████▉ | 700/782 [01:19<00:09,  8.86it/s, loss=0.284, v_num=1, val_loss=0.434, val_acc=0.855]\n",
      "Epoch 28:  91%|█████████ | 710/782 [01:19<00:08,  8.94it/s, loss=0.284, v_num=1, val_loss=0.434, val_acc=0.855]\n",
      "Epoch 28:  92%|█████████▏| 720/782 [01:19<00:06,  9.03it/s, loss=0.284, v_num=1, val_loss=0.434, val_acc=0.855]\n",
      "Epoch 28:  93%|█████████▎| 730/782 [01:20<00:05,  9.12it/s, loss=0.284, v_num=1, val_loss=0.434, val_acc=0.855]\n",
      "Epoch 28:  95%|█████████▍| 740/782 [01:20<00:04,  9.21it/s, loss=0.284, v_num=1, val_loss=0.434, val_acc=0.855]\n",
      "Epoch 28:  96%|█████████▌| 750/782 [01:20<00:03,  9.29it/s, loss=0.284, v_num=1, val_loss=0.434, val_acc=0.855]\n",
      "Epoch 28:  97%|█████████▋| 760/782 [01:21<00:02,  9.38it/s, loss=0.284, v_num=1, val_loss=0.434, val_acc=0.855]\n",
      "Epoch 28:  98%|█████████▊| 770/782 [01:21<00:01,  9.46it/s, loss=0.284, v_num=1, val_loss=0.434, val_acc=0.855]\n",
      "Epoch 28: 100%|█████████▉| 780/782 [01:21<00:00,  9.55it/s, loss=0.284, v_num=1, val_loss=0.434, val_acc=0.855]\n",
      "Epoch 28: 100%|██████████| 782/782 [01:22<00:00,  9.53it/s, loss=0.31, v_num=1, val_loss=0.350, val_acc=0.888] \n",
      "Epoch 29:  81%|████████  | 630/782 [01:24<00:20,  7.44it/s, loss=0.287, v_num=1, val_loss=0.350, val_acc=0.888]\n",
      "Validating: 0it [00:00, ?it/s]\u001b[A\n",
      "Validating:   0%|          | 0/157 [00:00<?, ?it/s]\u001b[A\n",
      "Epoch 29:  82%|████████▏ | 640/782 [01:25<00:18,  7.53it/s, loss=0.287, v_num=1, val_loss=0.350, val_acc=0.888]\n",
      "Epoch 29:  83%|████████▎ | 650/782 [01:25<00:17,  7.62it/s, loss=0.287, v_num=1, val_loss=0.350, val_acc=0.888]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29:  84%|████████▍ | 660/782 [01:25<00:15,  7.71it/s, loss=0.287, v_num=1, val_loss=0.350, val_acc=0.888]\n",
      "Epoch 29:  86%|████████▌ | 670/782 [01:26<00:14,  7.80it/s, loss=0.287, v_num=1, val_loss=0.350, val_acc=0.888]\n",
      "Epoch 29:  87%|████████▋ | 680/782 [01:26<00:12,  7.88it/s, loss=0.287, v_num=1, val_loss=0.350, val_acc=0.888]\n",
      "Epoch 29:  88%|████████▊ | 690/782 [01:26<00:11,  7.96it/s, loss=0.287, v_num=1, val_loss=0.350, val_acc=0.888]\n",
      "Epoch 29:  90%|████████▉ | 700/782 [01:27<00:10,  8.05it/s, loss=0.287, v_num=1, val_loss=0.350, val_acc=0.888]\n",
      "Epoch 29:  91%|█████████ | 710/782 [01:27<00:08,  8.13it/s, loss=0.287, v_num=1, val_loss=0.350, val_acc=0.888]\n",
      "Epoch 29:  92%|█████████▏| 720/782 [01:27<00:07,  8.21it/s, loss=0.287, v_num=1, val_loss=0.350, val_acc=0.888]\n",
      "Epoch 29:  93%|█████████▎| 730/782 [01:28<00:06,  8.29it/s, loss=0.287, v_num=1, val_loss=0.350, val_acc=0.888]\n",
      "Epoch 29:  95%|█████████▍| 740/782 [01:28<00:05,  8.38it/s, loss=0.287, v_num=1, val_loss=0.350, val_acc=0.888]\n",
      "Epoch 29:  96%|█████████▌| 750/782 [01:28<00:03,  8.46it/s, loss=0.287, v_num=1, val_loss=0.350, val_acc=0.888]\n",
      "Epoch 29:  97%|█████████▋| 760/782 [01:29<00:02,  8.53it/s, loss=0.287, v_num=1, val_loss=0.350, val_acc=0.888]\n",
      "Epoch 29:  98%|█████████▊| 770/782 [01:29<00:01,  8.61it/s, loss=0.287, v_num=1, val_loss=0.350, val_acc=0.888]\n",
      "Epoch 29: 100%|█████████▉| 780/782 [01:29<00:00,  8.68it/s, loss=0.287, v_num=1, val_loss=0.350, val_acc=0.888]\n",
      "Epoch 29: 100%|██████████| 782/782 [01:30<00:00,  8.67it/s, loss=0.267, v_num=1, val_loss=0.305, val_acc=0.895]\n",
      "Epoch 29: 100%|██████████| 782/782 [01:30<00:00,  8.66it/s, loss=0.267, v_num=1, val_loss=0.305, val_acc=0.895]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/plugins/training_type/ddp_spawn.py:288: UserWarning: cleaning up ddp environment...\n",
      "  rank_zero_warn(\"cleaning up ddp environment...\")\n",
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:106: UserWarning: The dataloader, train dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 96 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  f\"The dataloader, {name}, does not have many workers which may be a bottleneck.\"\n",
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/torch/optim/lr_scheduler.py:125: UserWarning: Seems like `optimizer.step()` has been overridden after learning rate scheduler initialization. Please, make sure to call `optimizer.step()` before `lr_scheduler.step()`. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\n",
      "  \"https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate\", UserWarning)\n",
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/plugins/training_type/ddp_spawn.py:288: UserWarning: cleaning up ddp environment...\n",
      "  rank_zero_warn(\"cleaning up ddp environment...\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[WARNING] \"import intel_pytorch_extension\" will be deprecated in future releases. Please use \"import torch_ipex\" instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/core/datamodule.py:424: LightningDeprecationWarning: DataModule.teardown has already been called, so it will not be called again. In v1.6 this behavior will change to always call DataModule.teardown.\n",
      "  f\"DataModule.{name} has already been called, so it will not be called again. \"\n",
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/core/datamodule.py:424: LightningDeprecationWarning: DataModule.setup has already been called, so it will not be called again. In v1.6 this behavior will change to always call DataModule.setup.\n",
      "  f\"DataModule.{name} has already been called, so it will not be called again. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "evaluate\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Global seed set to 7\n",
      "initializing ddp: GLOBAL_RANK: 0, MEMBER: 1/2\n",
      "Global seed set to 7\n",
      "initializing ddp: GLOBAL_RANK: 1, MEMBER: 2/2\n",
      "----------------------------------------------------------------------------------------------------\n",
      "distributed_backend=gloo\n",
      "All DDP processes registered. Starting ddp with 2 processes\n",
      "----------------------------------------------------------------------------------------------------\n",
      "\n",
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:106: UserWarning: The dataloader, test dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 96 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  f\"The dataloader, {name}, does not have many workers which may be a bottleneck.\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[WARNING] \"import intel_pytorch_extension\" will be deprecated in future releases. Please use \"import torch_ipex\" instead.\n",
      "Testing: 100%|██████████| 157/157 [00:06<00:00, 25.05it/s]--------------------------------------------------------------------------------\n",
      "DATALOADER:0 TEST RESULTS\n",
      "{'test_acc': 0.8917999863624573, 'test_loss': 0.3283665180206299}\n",
      "--------------------------------------------------------------------------------\n",
      "Testing: 100%|██████████| 157/157 [00:06<00:00, 22.51it/s][WARNING] \"import intel_pytorch_extension\" will be deprecated in future releases. Please use \"import torch_ipex\" instead.\n",
      "--------------------------------------------------------------------------------\n",
      "DATALOADER:0 TEST RESULTS\n",
      "{'test_acc': 0.8917999863624573, 'test_loss': 0.3283665180206299}\n",
      "--------------------------------------------------------------------------------\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/trainer/data_loading.py:106: UserWarning: The dataloader, test dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 96 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  f\"The dataloader, {name}, does not have many workers which may be a bottleneck.\"\n",
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/plugins/training_type/ddp_spawn.py:288: UserWarning: cleaning up ddp environment...\n",
      "  rank_zero_warn(\"cleaning up ddp environment...\")\n",
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/plugins/training_type/ddp_spawn.py:288: UserWarning: cleaning up ddp environment...\n",
      "  rank_zero_warn(\"cleaning up ddp environment...\")\n",
      "/home/mingzhi/anaconda3/envs/nanoWithPytorch/lib/python3.7/site-packages/pytorch_lightning/core/datamodule.py:424: LightningDeprecationWarning: DataModule.teardown has already been called, so it will not be called again. In v1.6 this behavior will change to always call DataModule.teardown.\n",
      "  f\"DataModule.{name} has already been called, so it will not be called again. \"\n"
     ]
    }
   ],
   "source": [
    "model = LitResnet(learning_rate=0.1, batch_size=64)\n",
    "model.datamodule = data_module\n",
    "checkpoint_callback = ModelCheckpoint(dirpath=\"checkpoints/\", save_top_k=1, monitor=\"val_loss\", filename=\"renet18_multi_ipex\", save_weights_only=True)\n",
    "multi_ipex_trainer = Trainer(num_processes=2,\n",
    "                       use_ipex=True,\n",
    "                       distributed_backend=\"subprocess\",\n",
    "                       progress_bar_refresh_rate=10,\n",
    "                       max_epochs=EPOCHS,\n",
    "                       fast_dev_run=DEV_RUN,\n",
    "                       logger=TensorBoardLogger(\"lightning_logs/\", name=\"multi_ipx\"),\n",
    "                       callbacks=[LearningRateMonitor(logging_interval=\"step\"), checkpoint_callback])\n",
    "start = time()\n",
    "multi_ipex_trainer.fit(model, datamodule=data_module)\n",
    "multi_ipex_fit_time = time() - start\n",
    "outputs = multi_ipex_trainer.test(model, datamodule=data_module)\n",
    "multi_ipex_acc = outputs[0]['test_acc'] * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "|      Precision    | Fit Time(s)       | Accuracy(%) |\n",
      "|        Basic      |       4559.45       |    90.29    |\n",
      "|  Single With Ipex |       4380.92       |    89.94    |\n",
      "| Multiple With Ipex|       2633.85       |    89.18    |\n",
      "\n"
     ]
    }
   ],
   "source": [
    "template = \"\"\"\n",
    "|      Precision    | Fit Time(s)       | Accuracy(%) |\n",
    "|        Basic      |       {:5.2f}       |    {:5.2f}    |\n",
    "|  Single With Ipex |       {:5.2f}       |    {:5.2f}    |\n",
    "| Multiple With Ipex|       {:5.2f}       |    {:5.2f}    |\n",
    "\"\"\"\n",
    "summary = template.format(\n",
    "    basic_fit_time, basic_acc,\n",
    "    single_ipex_fit_time, single_ipex_acc,\n",
    "    multi_ipex_fit_time, multi_ipex_acc\n",
    ")\n",
    "print(summary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.10 ('nanoPytorch')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "vscode": {
   "interpreter": {
    "hash": "17e9b820e65205a719775bb1f6bc42a5bd2579dee3e04b83232f9b833a088bb8"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
